[dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. It also improves the debug 
output in esp.c, sa.c and ipsec-secgw.c 

Bernard Iremonger (6):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw: fix 1st packet dropped patch two
  examples/ipsec-secgw: fix 1st packet dropped patch three
  examples/ipsec-secgw: fix debug in esp.c
  examples/ipsec-secgw: fix debug in sa.c
  examples/ipsec-secgw: fix debug in ipsec-secgw.c

 examples/ipsec-secgw/esp.c           |   5 +-
 examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++-----------------
 examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  67 ++++++++--
 6 files changed, 274 insertions(+), 187 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH 1/6] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Refactor create_session() into create_inline_session() and
create_lookaside_session() in ipsec.c
Use socket_ctx in create_inline_session()

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec.c         | 123 +++++++++++++++++++++++------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +--
 3 files changed, 90 insertions(+), 47 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..7090dbe 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,82 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if ((sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+				(sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)) {
+					RTE_LOG(ERR, IPSEC,
+						"Inline not supported\n");
+					return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +206,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +255,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +311,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +338,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +375,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +445,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +464,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +479,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +487,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..db5a5c1 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -306,6 +306,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index e403c46..152a684 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
-- 
2.7.4

[dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Bernard Iremonger]

Call create_inline_session() at initialisition in sa.c
Call rte_ipsec_session_prepare() in fill_ipsec_session() for inline.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/sa.c | 46 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 414fcd2..7fb1929 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH 3/6] examples/ipsec-secgw: fix 1st packet dropped patch three

[Bernard Iremonger]

Refactor cryprodev_init() and main() in ipsec-secgw.c
Add max_session_size().
Start ports before adding flows in main().

Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c | 246 ++++++++++++++++++-------------------
 1 file changed, 123 insertions(+), 123 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 8e7cd1b..805e6b4 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1029,6 +1029,9 @@ main_loop(__attribute__((unused)) void *dummy)
 				drain_outbound_crypto_queues(qconf,
 					&qconf->outbound);
 		}
+
+		drain_inbound_crypto_queues(qconf, &qconf->inbound);
+		drain_outbound_crypto_queues(qconf, &qconf->outbound);
 	}
 }
 
@@ -1627,7 +1630,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1650,45 +1653,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1726,45 +1690,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1785,38 +1710,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1982,6 +1875,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2062,9 +2048,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2092,7 +2080,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2102,20 +2091,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2133,7 +2116,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2151,6 +2138,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
-- 
2.7.4

[dpdk-dev] [PATCH 4/6] examples/ipsec-secgw: fix debug in esp.c

[Bernard Iremonger]

Improve debug code in esp.c

Fixes: f159e70b0922 ("examples/ipsec-secgw: support transport mode")
Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/esp.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index e33232c..faa84dd 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -162,7 +162,7 @@ esp_inbound_post(struct rte_mbuf *m, struct ipsec_sa *sa,
 	}
 
 	if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
-		RTE_LOG(ERR, IPSEC_ESP, "failed crypto op\n");
+		RTE_LOG(ERR, IPSEC_ESP, "%s() failed crypto op\n", __func__);
 		return -1;
 	}
 
@@ -455,7 +455,8 @@ esp_outbound_post(struct rte_mbuf *m,
 	} else {
 		RTE_ASSERT(cop != NULL);
 		if (cop->status != RTE_CRYPTO_OP_STATUS_SUCCESS) {
-			RTE_LOG(ERR, IPSEC_ESP, "Failed crypto op\n");
+			RTE_LOG(ERR, IPSEC_ESP, "%s() failed crypto op\n",
+				__func__);
 			return -1;
 		}
 	}
-- 
2.7.4

[dpdk-dev] [PATCH 5/6] examples/ipsec-secgw: fix debug in sa.c

[Bernard Iremonger]

Improve debug code in sa.c

Fixes: 0d547ed03717 ("examples/ipsec-secgw: support configuration file")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/sa.c | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 7fb1929..4e583cd 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -688,7 +688,22 @@ print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 		}
 		break;
 	case TRANSPORT:
-		printf("Transport");
+		printf("Transport ");
+		break;
+	}
+	printf(" type:");
+	switch (sa->type) {
+	case RTE_SECURITY_ACTION_TYPE_NONE:
+		printf("no-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+		printf("inline-crypto-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
+		printf("inline-protocol-offload ");
+		break;
+	case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
+		printf("lookaside-protocol-offload ");
 		break;
 	}
 	printf("\n");
@@ -716,8 +731,8 @@ sa_create(const char *name, int32_t socket_id)
 	snprintf(s, sizeof(s), "%s_%u", name, socket_id);
 
 	/* Create SA array table */
-	printf("Creating SA context with %u maximum entries\n",
-			IPSEC_SA_MAX_ENTRIES);
+	printf("Creating SA context with %u maximum entries on socket %d\n",
+			IPSEC_SA_MAX_ENTRIES, socket_id);
 
 	mz_size = sizeof(struct sa_ctx);
 	mz = rte_memzone_reserve(s, mz_size, socket_id,
-- 
2.7.4

[dpdk-dev] [PATCH 6/6] examples/ipsec-secgw: fix debug in ipsec-secgw.c

[Bernard Iremonger]

Improve debug in ipsec-secgw.c

Fixes: 906257e965b7 ("examples/ipsec-secgw: support IPv6")
Cc: stable@dpdk.org
Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 805e6b4..0c3c8f8 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -260,7 +260,8 @@ prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 		pkt->l3_len = sizeof(struct ip6_hdr);
 	} else {
 		/* Unknown/Unsupported type, drop the packet */
-		RTE_LOG(ERR, IPSEC, "Unsupported packet type\n");
+		RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
+			rte_be_to_cpu_16(eth->ether_type));
 		rte_pktmbuf_free(pkt);
 	}
 
@@ -984,7 +985,8 @@ main_loop(__attribute__((unused)) void *dummy)
 			socket_ctx[socket_id].session_priv_pool;
 
 	if (qconf->nb_rx_queue == 0) {
-		RTE_LOG(INFO, IPSEC, "lcore %u has nothing to do\n", lcore_id);
+		RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n",
+			lcore_id);
 		return 0;
 	}
 
-- 
2.7.4

Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Akhil Goyal]

Hi Bernard,

On 3/6/2019 9:30 PM, Bernard Iremonger wrote:
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto. It also improves the debug
> output in esp.c, sa.c and ipsec-secgw.c
>
> Bernard Iremonger (6):
>    examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>    examples/ipsec-secgw: fix 1st packet dropped patch two
>    examples/ipsec-secgw: fix 1st packet dropped patch three
patch titles are not good. You can probably add some logical comments in 
the description.
you may also consider squashing the patches if no logical bifurcation 
can be made.

>    examples/ipsec-secgw: fix debug in esp.c
>    examples/ipsec-secgw: fix debug in sa.c
>    examples/ipsec-secgw: fix debug in ipsec-secgw.c
Above 3 patches can be squashed and can be separated from this series.
>   examples/ipsec-secgw/esp.c           |   5 +-
>   examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++-----------------
>   examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  67 ++++++++--
>   6 files changed, 274 insertions(+), 187 deletions(-)
>

Re: [dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Ananyev, Konstantin]

Hi Bernard,

> 
> Call create_inline_session() at initialisition in sa.c
> Call rte_ipsec_session_prepare() in fill_ipsec_session() for inline.

Here and in other places - it probably worth to explain what is the purpose
for  these changes. 
As a side notice, as these series fixes that problem, it probably worse to add a patch
into series that removes the following:

       # to overcome problem with ipsec-secgw for inline mode,
        # when first packet(s) will be always dropped.
        # note that ping will fail here
        ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}

from examples/ipsec-secgw/test/(tun|trs)_aesgcm_defs.sh
Konstantin

> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Cc: stable@dpdk.org
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/sa.c | 46 ++++++++++++++++++++++++++++++++++++----------
>  1 file changed, 36 insertions(+), 10 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index 414fcd2..7fb1929 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
> 
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +		uint32_t nb_entries, uint32_t inbound,
> +		struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  			sa->xforms = &sa_ctx->xf[idx].a;
> 
> +			if (sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> +				sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +				rc = create_inline_session(skt_ctx, sa);
> +				if (rc != 0) {
> +					RTE_LOG(ERR, IPSEC_ESP,
> +						"create_inline_session() failed\n");
> +					return -EINVAL;
> +				}
> +			}
>  			print_one_sa_rule(sa, inbound);
>  		} else {
>  			switch (sa->cipher_algo) {
> @@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> +		if (ss->security.ses != NULL) {
> +			rc = rte_ipsec_session_prepare(ss);
> +			if (rc != 0)
> +				memset(ss, 0, sizeof(*ss));
> +		}
> +	}
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch two

[Iremonger, Bernard]

Hi Konstantin,

<snip>

> Subject: RE: [PATCH 2/6] examples/ipsec-secgw: fix 1st packet dropped patch
> two
> 
> 
> Hi Bernard,
> 
> >
> > Call create_inline_session() at initialisition in sa.c Call
> > rte_ipsec_session_prepare() in fill_ipsec_session() for inline.
> 
> Here and in other places - it probably worth to explain what is the purpose for
> these changes.

I will improve explanation.

> As a side notice, as these series fixes that problem, it probably worse to add a
> patch into series that removes the following:
> 
>        # to overcome problem with ipsec-secgw for inline mode,
>         # when first packet(s) will be always dropped.
>         # note that ping will fail here
>         ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
> 
> from examples/ipsec-secgw/test/(tun|trs)_aesgcm_defs.sh
> Konstantin

I will add a patch to fix the test scripts.

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>


> Subject: Re: [dpdk-dev] [PATCH 0/6] examples/ipsec-secgw: fix 1st pkt dropped
> 
> Hi Bernard,
> 
> On 3/6/2019 9:30 PM, Bernard Iremonger wrote:
> > This patchset fixes the issue of the first inbound packet being
> > dropped for inline crypto. It also improves the debug output in esp.c,
> > sa.c and ipsec-secgw.c
> >
> > Bernard Iremonger (6):
> >    examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >    examples/ipsec-secgw: fix 1st packet dropped patch two
> >    examples/ipsec-secgw: fix 1st packet dropped patch three
> patch titles are not good. You can probably add some logical comments in the
> description.

I will update the patch titles and descriptions.

> you may also consider squashing the patches if no logical bifurcation can be
> made.

I thought it might be easier to review the changes as separate patches.
If you prefer one patch I can squash them.

> >    examples/ipsec-secgw: fix debug in esp.c
> >    examples/ipsec-secgw: fix debug in sa.c
> >    examples/ipsec-secgw: fix debug in ipsec-secgw.c
> Above 3 patches can be squashed and can be separated from this series.

I will remove from this patch set, squash and send as an independent patch.  

> >   examples/ipsec-secgw/esp.c           |   5 +-
> >   examples/ipsec-secgw/ipsec-secgw.c   | 252 ++++++++++++++++++------------
> -----
> >   examples/ipsec-secgw/ipsec.c         | 123 +++++++++++------
> >   examples/ipsec-secgw/ipsec.h         |   5 +-
> >   examples/ipsec-secgw/ipsec_process.c |   9 +-
> >   examples/ipsec-secgw/sa.c            |  67 ++++++++--
> >   6 files changed, 274 insertions(+), 187 deletions(-)
> >

Regards,

Bernard.

[dpdk-dev] [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
 examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++++--
 5 files changed, 246 insertions(+), 180 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 8e7cd1b..835cc98 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1627,7 +1627,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1650,45 +1650,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1726,45 +1687,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1785,38 +1707,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1982,6 +1872,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2062,9 +2045,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2092,7 +2077,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2102,20 +2088,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2133,7 +2113,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2151,6 +2135,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..7090dbe 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,82 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if ((sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+				(sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)) {
+					RTE_LOG(ERR, IPSEC,
+						"Inline not supported\n");
+					return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +206,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +255,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +311,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +338,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +375,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +445,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +464,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +479,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +487,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..db5a5c1 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -306,6 +306,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index e403c46..152a684 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 414fcd2..7fb1929 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -762,11 +762,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -819,6 +821,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -894,16 +907,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -997,10 +1010,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1013,6 +1028,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1047,8 +1073,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1126,7 +1152,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1146,7 +1172,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v2 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Thursday, March 7, 2019 2:58 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>
> Subject: [PATCH v2 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v2:
> -------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
>  examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 +++--
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
>  7 files changed, 246 insertions(+), 200 deletions(-)
> 
> --

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
>
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
why do we need a separate function for session creation for inline and 
lookaside cases?
Why can't we initialize the sessions on sa_init in both the cases?
>
> The max_session_size() function has been added to calculate memory
> requirements.
>
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
>
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
>
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
>
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
>
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>   examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
>   examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  46 +++++--
>   5 files changed, 246 insertions(+), 180 deletions(-)
>
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
>
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
why do we need a separate function for session creation for inline and 
lookaside cases?
Why can't we initialize the sessions on sa_init in both the cases?
>
> The max_session_size() function has been added to calculate memory
> requirements.
>
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
>
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
>
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
>
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
>
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>   examples/ipsec-secgw/ipsec-secgw.c   | 243 +++++++++++++++++------------------
>   examples/ipsec-secgw/ipsec.c         | 123 ++++++++++++------
>   examples/ipsec-secgw/ipsec.h         |   5 +-
>   examples/ipsec-secgw/ipsec_process.c |   9 +-
>   examples/ipsec-secgw/sa.c            |  46 +++++--
>   5 files changed, 246 insertions(+), 180 deletions(-)
>
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
> dropped for inline crypto
> 
> Hi Bernard,
> 
> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> why do we need a separate function for session creation for inline and
> lookaside cases?
> Why can't we initialize the sessions on sa_init in both the cases?

For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
all of the required data is available in the in the skt_ctx variable.
The skt_ctx variable is already setup when sa_init() is called.

For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
the required data is available in the ipsec_ctx variable.

The ipsec_ctx variable is not setup when sa_init() is called.
It is setup in the main_loop() function  when the variable qconf is setup.
The main_loop() function is called after the sa_init() function is called.

I hope this  answers your question

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
> dropped for inline crypto
> 
> Hi Bernard,
> 
> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> why do we need a separate function for session creation for inline and
> lookaside cases?
> Why can't we initialize the sessions on sa_init in both the cases?

For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
all of the required data is available in the in the skt_ctx variable.
The skt_ctx variable is already setup when sa_init() is called.

For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
the required data is available in the ipsec_ctx variable.

The ipsec_ctx variable is not setup when sa_init() is called.
It is setup in the main_loop() function  when the variable qconf is setup.
The main_loop() function is called after the sa_init() function is called.

I hope this  answers your question

<snip>

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 3:52 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
>> dropped for inline crypto
>>
>> Hi Bernard,
>>
>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>> Inline crypto installs a flow rule in the NIC. This flow rule must be
>>> installed before the first inbound packet is received.
>>>
>>> The create_session() function installs the flow rule,
>>> create_session() has been refactored into create_inline_session() and
>>> create_lookaside_session(). The create_inline_session() function uses
>>> the socket_ctx data and is now called at initialisation in
>>> sa_add_rules().
>> why do we need a separate function for session creation for inline and
>> lookaside cases?
>> Why can't we initialize the sessions on sa_init in both the cases?
> For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
> all of the required data is available in the in the skt_ctx variable.
> The skt_ctx variable is already setup when sa_init() is called.
>
> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
> the required data is available in the ipsec_ctx variable.
>
> The ipsec_ctx variable is not setup when sa_init() is called.
> It is setup in the main_loop() function  when the variable qconf is setup.
> The main_loop() function is called after the sa_init() function is called.
>
> I hope this  answers your question
Whatever information that is required for session creation is available 
before we call the main loop() in both the cases.
My point is both the sessions(inline/lookaside) can be init at the same 
position, we do not need to have a separate path for them.
If it is not possible in sa_init(), it may be somewhere else before the 
actual data path is started.

The problem with inline processing is that, h/w need to know the SA 
before the first packet is received. So we cannot init the session on 
receive of first packet. However there is no such limitation in case of 
lookaside, it can be initialized anywhere.

-Akhil

>
> <snip>
>
> Regards,
>
> Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 3:52 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt
>> dropped for inline crypto
>>
>> Hi Bernard,
>>
>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>> Inline crypto installs a flow rule in the NIC. This flow rule must be
>>> installed before the first inbound packet is received.
>>>
>>> The create_session() function installs the flow rule,
>>> create_session() has been refactored into create_inline_session() and
>>> create_lookaside_session(). The create_inline_session() function uses
>>> the socket_ctx data and is now called at initialisation in
>>> sa_add_rules().
>> why do we need a separate function for session creation for inline and
>> lookaside cases?
>> Why can't we initialize the sessions on sa_init in both the cases?
> For the create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa) function,
> all of the required data is available in the in the skt_ctx variable.
> The skt_ctx variable is already setup when sa_init() is called.
>
> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) function,
> the required data is available in the ipsec_ctx variable.
>
> The ipsec_ctx variable is not setup when sa_init() is called.
> It is setup in the main_loop() function  when the variable qconf is setup.
> The main_loop() function is called after the sa_init() function is called.
>
> I hope this  answers your question
Whatever information that is required for session creation is available 
before we call the main loop() in both the cases.
My point is both the sessions(inline/lookaside) can be init at the same 
position, we do not need to have a separate path for them.
If it is not possible in sa_init(), it may be somewhere else before the 
actual data path is started.

The problem with inline processing is that, h/w need to know the SA 
before the first packet is received. So we cannot init the session on 
receive of first packet. However there is no such limitation in case of 
lookaside, it can be initialized anywhere.

-Akhil

>
> <snip>
>
> Regards,
>
> Bernard.

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> >> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >> pkt dropped for inline crypto
> >>
> >> Hi Bernard,
> >>
> >> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>> be installed before the first inbound packet is received.
> >>>
> >>> The create_session() function installs the flow rule,
> >>> create_session() has been refactored into create_inline_session()
> >>> and create_lookaside_session(). The create_inline_session() function
> >>> uses the socket_ctx data and is now called at initialisation in
> >>> sa_add_rules().
> >> why do we need a separate function for session creation for inline
> >> and lookaside cases?
> >> Why can't we initialize the sessions on sa_init in both the cases?
> > For the create_inline_session(struct socket_ctx *skt_ctx, struct
> > ipsec_sa *sa) function, all of the required data is available in the in the
> skt_ctx variable.
> > The skt_ctx variable is already setup when sa_init() is called.
> >
> > For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> variable.
> >
> > The ipsec_ctx variable is not setup when sa_init() is called.
> > It is setup in the main_loop() function  when the variable qconf is setup.
> > The main_loop() function is called after the sa_init() function is called.
> >
> > I hope this  answers your question
> Whatever information that is required for session creation is available before
> we call the main loop() in both the cases.
> My point is both the sessions(inline/lookaside) can be init at the same
> position, we do not need to have a separate path for them.
> If it is not possible in sa_init(), it may be somewhere else before the actual
> data path is started.
> 
> The problem with inline processing is that, h/w need to know the SA before
> the first packet is received. So we cannot init the session on receive of first
> packet. However there is no such limitation in case of lookaside, it can be
> initialized anywhere.
> 
> -Akhil

This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.

It is not intended to refactor the lookaside processing.

Regards,

Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

<snip>

> >> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >> pkt dropped for inline crypto
> >>
> >> Hi Bernard,
> >>
> >> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>> be installed before the first inbound packet is received.
> >>>
> >>> The create_session() function installs the flow rule,
> >>> create_session() has been refactored into create_inline_session()
> >>> and create_lookaside_session(). The create_inline_session() function
> >>> uses the socket_ctx data and is now called at initialisation in
> >>> sa_add_rules().
> >> why do we need a separate function for session creation for inline
> >> and lookaside cases?
> >> Why can't we initialize the sessions on sa_init in both the cases?
> > For the create_inline_session(struct socket_ctx *skt_ctx, struct
> > ipsec_sa *sa) function, all of the required data is available in the in the
> skt_ctx variable.
> > The skt_ctx variable is already setup when sa_init() is called.
> >
> > For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> > ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> variable.
> >
> > The ipsec_ctx variable is not setup when sa_init() is called.
> > It is setup in the main_loop() function  when the variable qconf is setup.
> > The main_loop() function is called after the sa_init() function is called.
> >
> > I hope this  answers your question
> Whatever information that is required for session creation is available before
> we call the main loop() in both the cases.
> My point is both the sessions(inline/lookaside) can be init at the same
> position, we do not need to have a separate path for them.
> If it is not possible in sa_init(), it may be somewhere else before the actual
> data path is started.
> 
> The problem with inline processing is that, h/w need to know the SA before
> the first packet is received. So we cannot init the session on receive of first
> packet. However there is no such limitation in case of lookaside, it can be
> initialized anywhere.
> 
> -Akhil

This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.

It is not intended to refactor the lookaside processing.

Regards,

Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 5:11 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>
>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>> pkt dropped for inline crypto
>>>>
>>>> Hi Bernard,
>>>>
>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>> be installed before the first inbound packet is received.
>>>>>
>>>>> The create_session() function installs the flow rule,
>>>>> create_session() has been refactored into create_inline_session()
>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>> sa_add_rules().
>>>> why do we need a separate function for session creation for inline
>>>> and lookaside cases?
>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>> ipsec_sa *sa) function, all of the required data is available in the in the
>> skt_ctx variable.
>>> The skt_ctx variable is already setup when sa_init() is called.
>>>
>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>> variable.
>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>> The main_loop() function is called after the sa_init() function is called.
>>>
>>> I hope this  answers your question
>> Whatever information that is required for session creation is available before
>> we call the main loop() in both the cases.
>> My point is both the sessions(inline/lookaside) can be init at the same
>> position, we do not need to have a separate path for them.
>> If it is not possible in sa_init(), it may be somewhere else before the actual
>> data path is started.
>>
>> The problem with inline processing is that, h/w need to know the SA before
>> the first packet is received. So we cannot init the session on receive of first
>> packet. However there is no such limitation in case of lookaside, it can be
>> initialized anywhere.
>>
>> -Akhil
> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.
>
> It is not intended to refactor the lookaside processing.
The fix provided should not make the code complex and difficult to 
understand when we have the option available to fix that in a simpler 
way. Application is already complex enough(4 action types) and by adding 
separate code for session init at two different places will be difficult 
to maintain in future and will reduce the code readability.

-Akhil

> Regards,
>
> Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

On 3/26/2019 5:11 PM, Iremonger, Bernard wrote:
> Hi Akhil,
>
> <snip>
>
>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>> pkt dropped for inline crypto
>>>>
>>>> Hi Bernard,
>>>>
>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>> be installed before the first inbound packet is received.
>>>>>
>>>>> The create_session() function installs the flow rule,
>>>>> create_session() has been refactored into create_inline_session()
>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>> sa_add_rules().
>>>> why do we need a separate function for session creation for inline
>>>> and lookaside cases?
>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>> ipsec_sa *sa) function, all of the required data is available in the in the
>> skt_ctx variable.
>>> The skt_ctx variable is already setup when sa_init() is called.
>>>
>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>> variable.
>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>> The main_loop() function is called after the sa_init() function is called.
>>>
>>> I hope this  answers your question
>> Whatever information that is required for session creation is available before
>> we call the main loop() in both the cases.
>> My point is both the sessions(inline/lookaside) can be init at the same
>> position, we do not need to have a separate path for them.
>> If it is not possible in sa_init(), it may be somewhere else before the actual
>> data path is started.
>>
>> The problem with inline processing is that, h/w need to know the SA before
>> the first packet is received. So we cannot init the session on receive of first
>> packet. However there is no such limitation in case of lookaside, it can be
>> initialized anywhere.
>>
>> -Akhil
> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is received while leaving the lookaside processing as it was originally.
>
> It is not intended to refactor the lookaside processing.
The fix provided should not make the code complex and difficult to 
understand when we have the option available to fix that in a simpler 
way. Application is already complex enough(4 action types) and by adding 
separate code for session init at two different places will be difficult 
to maintain in future and will reduce the code readability.

-Akhil

> Regards,
>
> Bernard

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> >
> >>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >>>> pkt dropped for inline crypto
> >>>>
> >>>> Hi Bernard,
> >>>>
> >>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>>>> be installed before the first inbound packet is received.
> >>>>>
> >>>>> The create_session() function installs the flow rule,
> >>>>> create_session() has been refactored into create_inline_session()
> >>>>> and create_lookaside_session(). The create_inline_session() function
> >>>>> uses the socket_ctx data and is now called at initialisation in
> >>>>> sa_add_rules().
> >>>> why do we need a separate function for session creation for inline
> >>>> and lookaside cases?
> >>>> Why can't we initialize the sessions on sa_init in both the cases?
> >>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
> >>> ipsec_sa *sa) function, all of the required data is available in the in the
> >> skt_ctx variable.
> >>> The skt_ctx variable is already setup when sa_init() is called.
> >>>
> >>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> >>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> >> variable.
> >>> The ipsec_ctx variable is not setup when sa_init() is called.
> >>> It is setup in the main_loop() function  when the variable qconf is setup.
> >>> The main_loop() function is called after the sa_init() function is called.
> >>>
> >>> I hope this  answers your question
> >> Whatever information that is required for session creation is available before
> >> we call the main loop() in both the cases.
> >> My point is both the sessions(inline/lookaside) can be init at the same
> >> position, we do not need to have a separate path for them.
> >> If it is not possible in sa_init(), it may be somewhere else before the actual
> >> data path is started.
> >>
> >> The problem with inline processing is that, h/w need to know the SA before
> >> the first packet is received. So we cannot init the session on receive of first
> >> packet. However there is no such limitation in case of lookaside, it can be
> >> initialized anywhere.
> >>
> >> -Akhil
> > This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
> received while leaving the lookaside processing as it was originally.
> >
> > It is not intended to refactor the lookaside processing.
> The fix provided should not make the code complex and difficult to
> understand when we have the option available to fix that in a simpler
> way. Application is already complex enough(4 action types) and by adding
> separate code for session init at two different places will be difficult
> to maintain in future and will reduce the code readability.

It can be changed to do initialization for both security and crypto sessions at sa_init()
time, but it would cause much more changes in the code:
the idea is for each SA go through all available crypto-devices, check their capabilities,
and if they match, we'll invoke init_session() for that SA and device.
That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
crypto-dev.
As a drawback -  each session might occupy more memory.
Though as I said, it would require more changes (and testing) in init code.
So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
What do you think?
Konstantin

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Akhil,

> >
> >>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
> >>>> pkt dropped for inline crypto
> >>>>
> >>>> Hi Bernard,
> >>>>
> >>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
> >>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
> >>>>> be installed before the first inbound packet is received.
> >>>>>
> >>>>> The create_session() function installs the flow rule,
> >>>>> create_session() has been refactored into create_inline_session()
> >>>>> and create_lookaside_session(). The create_inline_session() function
> >>>>> uses the socket_ctx data and is now called at initialisation in
> >>>>> sa_add_rules().
> >>>> why do we need a separate function for session creation for inline
> >>>> and lookaside cases?
> >>>> Why can't we initialize the sessions on sa_init in both the cases?
> >>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
> >>> ipsec_sa *sa) function, all of the required data is available in the in the
> >> skt_ctx variable.
> >>> The skt_ctx variable is already setup when sa_init() is called.
> >>>
> >>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
> >>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
> >> variable.
> >>> The ipsec_ctx variable is not setup when sa_init() is called.
> >>> It is setup in the main_loop() function  when the variable qconf is setup.
> >>> The main_loop() function is called after the sa_init() function is called.
> >>>
> >>> I hope this  answers your question
> >> Whatever information that is required for session creation is available before
> >> we call the main loop() in both the cases.
> >> My point is both the sessions(inline/lookaside) can be init at the same
> >> position, we do not need to have a separate path for them.
> >> If it is not possible in sa_init(), it may be somewhere else before the actual
> >> data path is started.
> >>
> >> The problem with inline processing is that, h/w need to know the SA before
> >> the first packet is received. So we cannot init the session on receive of first
> >> packet. However there is no such limitation in case of lookaside, it can be
> >> initialized anywhere.
> >>
> >> -Akhil
> > This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
> received while leaving the lookaside processing as it was originally.
> >
> > It is not intended to refactor the lookaside processing.
> The fix provided should not make the code complex and difficult to
> understand when we have the option available to fix that in a simpler
> way. Application is already complex enough(4 action types) and by adding
> separate code for session init at two different places will be difficult
> to maintain in future and will reduce the code readability.

It can be changed to do initialization for both security and crypto sessions at sa_init()
time, but it would cause much more changes in the code:
the idea is for each SA go through all available crypto-devices, check their capabilities,
and if they match, we'll invoke init_session() for that SA and device.
That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
crypto-dev.
As a drawback -  each session might occupy more memory.
Though as I said, it would require more changes (and testing) in init code.
So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
What do you think?
Konstantin

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

On 3/26/2019 5:59 PM, Ananyev, Konstantin wrote:
> Hi Akhil,
>
>>>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>>>> pkt dropped for inline crypto
>>>>>>
>>>>>> Hi Bernard,
>>>>>>
>>>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>>>> be installed before the first inbound packet is received.
>>>>>>>
>>>>>>> The create_session() function installs the flow rule,
>>>>>>> create_session() has been refactored into create_inline_session()
>>>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>>>> sa_add_rules().
>>>>>> why do we need a separate function for session creation for inline
>>>>>> and lookaside cases?
>>>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>>>> ipsec_sa *sa) function, all of the required data is available in the in the
>>>> skt_ctx variable.
>>>>> The skt_ctx variable is already setup when sa_init() is called.
>>>>>
>>>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>>>> variable.
>>>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>>>> The main_loop() function is called after the sa_init() function is called.
>>>>>
>>>>> I hope this  answers your question
>>>> Whatever information that is required for session creation is available before
>>>> we call the main loop() in both the cases.
>>>> My point is both the sessions(inline/lookaside) can be init at the same
>>>> position, we do not need to have a separate path for them.
>>>> If it is not possible in sa_init(), it may be somewhere else before the actual
>>>> data path is started.
>>>>
>>>> The problem with inline processing is that, h/w need to know the SA before
>>>> the first packet is received. So we cannot init the session on receive of first
>>>> packet. However there is no such limitation in case of lookaside, it can be
>>>> initialized anywhere.
>>>>
>>>> -Akhil
>>> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
>> received while leaving the lookaside processing as it was originally.
>>> It is not intended to refactor the lookaside processing.
>> The fix provided should not make the code complex and difficult to
>> understand when we have the option available to fix that in a simpler
>> way. Application is already complex enough(4 action types) and by adding
>> separate code for session init at two different places will be difficult
>> to maintain in future and will reduce the code readability.
> It can be changed to do initialization for both security and crypto sessions at sa_init()
> time, but it would cause much more changes in the code:
> the idea is for each SA go through all available crypto-devices, check their capabilities,
> and if they match, we'll invoke init_session() for that SA and device.
> That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
> crypto-dev.
Agreed, my point is we should have same code path for all action types 
as much as possible to make the user comfortable to port their 
applications over DPDK.
> As a drawback -  each session might occupy more memory.
That is still there for the inline cases.
> Though as I said, it would require more changes (and testing) in init code.
> So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
> What do you think?
The code changes are still there in this patch as well. And the testing 
cycle has not started yet for this release.

5 files changed, 246 insertions(+), 180 deletions(-)

If we intend to defer it, we will have similar code movement again. So IMO, it is good that we do the changes in one go.
This is an application code and we can have it merged in RC2 as well.

> Konstantin
>

Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Konstantin,

On 3/26/2019 5:59 PM, Ananyev, Konstantin wrote:
> Hi Akhil,
>
>>>>>> Subject: Re: [dpdk-dev] [PATCH v2 1/2] examples/ipsec-secgw: fix 1st
>>>>>> pkt dropped for inline crypto
>>>>>>
>>>>>> Hi Bernard,
>>>>>>
>>>>>> On 3/7/2019 8:27 PM, Bernard Iremonger wrote:
>>>>>>> Inline crypto installs a flow rule in the NIC. This flow rule must
>>>>>>> be installed before the first inbound packet is received.
>>>>>>>
>>>>>>> The create_session() function installs the flow rule,
>>>>>>> create_session() has been refactored into create_inline_session()
>>>>>>> and create_lookaside_session(). The create_inline_session() function
>>>>>>> uses the socket_ctx data and is now called at initialisation in
>>>>>>> sa_add_rules().
>>>>>> why do we need a separate function for session creation for inline
>>>>>> and lookaside cases?
>>>>>> Why can't we initialize the sessions on sa_init in both the cases?
>>>>> For the create_inline_session(struct socket_ctx *skt_ctx, struct
>>>>> ipsec_sa *sa) function, all of the required data is available in the in the
>>>> skt_ctx variable.
>>>>> The skt_ctx variable is already setup when sa_init() is called.
>>>>>
>>>>> For the create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct
>>>>> ipsec_sa *sa) function, the required data is available in the ipsec_ctx
>>>> variable.
>>>>> The ipsec_ctx variable is not setup when sa_init() is called.
>>>>> It is setup in the main_loop() function  when the variable qconf is setup.
>>>>> The main_loop() function is called after the sa_init() function is called.
>>>>>
>>>>> I hope this  answers your question
>>>> Whatever information that is required for session creation is available before
>>>> we call the main loop() in both the cases.
>>>> My point is both the sessions(inline/lookaside) can be init at the same
>>>> position, we do not need to have a separate path for them.
>>>> If it is not possible in sa_init(), it may be somewhere else before the actual
>>>> data path is started.
>>>>
>>>> The problem with inline processing is that, h/w need to know the SA before
>>>> the first packet is received. So we cannot init the session on receive of first
>>>> packet. However there is no such limitation in case of lookaside, it can be
>>>> initialized anywhere.
>>>>
>>>> -Akhil
>>> This patch is intended to fix the bug in the inline processing, which is that  the flow rule must be installed, before the first packet is
>> received while leaving the lookaside processing as it was originally.
>>> It is not intended to refactor the lookaside processing.
>> The fix provided should not make the code complex and difficult to
>> understand when we have the option available to fix that in a simpler
>> way. Application is already complex enough(4 action types) and by adding
>> separate code for session init at two different places will be difficult
>> to maintain in future and will reduce the code readability.
> It can be changed to do initialization for both security and crypto sessions at sa_init()
> time, but it would cause much more changes in the code:
> the idea is for each SA go through all available crypto-devices, check their capabilities,
> and if they match, we'll invoke init_session() for that SA and device.
> That way at the end of sa_init() we'll have crypto-sessions that can be used on any attached
> crypto-dev.
Agreed, my point is we should have same code path for all action types 
as much as possible to make the user comfortable to port their 
applications over DPDK.
> As a drawback -  each session might occupy more memory.
That is still there for the inline cases.
> Though as I said, it would require more changes (and testing) in init code.
> So my thought to have this patch as it is and then add these changes as a separate patch series (phase 2).
> What do you think?
The code changes are still there in this patch as well. And the testing 
cycle has not started yet for this release.

5 files changed, 246 insertions(+), 180 deletions(-)

If we intend to defer it, we will have similar code movement again. So IMO, it is good that we do the changes in one go.
This is an application code and we can have it merged in RC2 as well.

> Konstantin
>

[dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2: 
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 495 insertions(+), 455 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2: 
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 495 insertions(+), 455 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
 examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 495 insertions(+), 435 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..e31d472 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
+				++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
 				action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
 examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 495 insertions(+), 435 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..e31d472 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
+				++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
 				action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v3 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v3 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v3:
> --------------
> The previous refactoring of the create_session() function has been dropped.
> The create_session() function is now called from sa_init() at startup.
> 
> The following functions have been added:
> crypto_devid_fill() in ipsec-secgw.c
> check_cryptodev_capability() in ipsec.c
> check_cryptodev_aead_capability() in ipsec.c
> create_sec_session() and create_crypto_session() in ipsec.c
> 
> The create_session() function has been refactored to call
> the create_sec_session() and create_crypto_session() functions.
> 
> 
> Changes in v2:
> --------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st packet dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
>  examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |  10 +-
>  examples/ipsec-secgw/ipsec_process.c         |  38 +-
>  examples/ipsec-secgw/sa.c                    |  42 +-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 495 insertions(+), 455 deletions(-)
> 
> --

As ipsec lib and these tests were introduced in 19.02,
I don't think you need to cc stable.
Apart from that:
Series Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v3 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Ananyev, Konstantin]

> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v3:
> --------------
> The previous refactoring of the create_session() function has been dropped.
> The create_session() function is now called from sa_init() at startup.
> 
> The following functions have been added:
> crypto_devid_fill() in ipsec-secgw.c
> check_cryptodev_capability() in ipsec.c
> check_cryptodev_aead_capability() in ipsec.c
> create_sec_session() and create_crypto_session() in ipsec.c
> 
> The create_session() function has been refactored to call
> the create_sec_session() and create_crypto_session() functions.
> 
> 
> Changes in v2:
> --------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st packet dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
>  examples/ipsec-secgw/ipsec.c                 | 569 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |  10 +-
>  examples/ipsec-secgw/ipsec_process.c         |  38 +-
>  examples/ipsec-secgw/sa.c                    |  42 +-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 495 insertions(+), 455 deletions(-)
> 
> --

As ipsec lib and these tests were introduced in 19.02,
I don't think you need to cc stable.
Apart from that:
Series Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule.
> 
> Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
> sessions at startup to fix the issue of the first packet
> being dropped for inline crypto.
> 
> The create_session() function is now called at initialisation in
> sa_add_rules() which is called from sa_init().
> The return code for add_rules is checked.
> Calls to create_session() in other functions are dropped.
> 
> Add crypto_devid_fill() in ipsec-secgw.c
> Add max_session_size() in ipsec-secgw.c
> Add check_cryptodev_capability() in ipsec.c
> Add check_cryptodev_aead_capability() in ipsec.c
> Add create_sec_session() and create_crypto_session() in ipsec.c
> 
> The crypto_dev_fill() function has been added to find the
> enabled crypto devices.
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The check_cryptodev_capability() and check_cryptodev_aead_capability()
> functions have been added to check that the SA is supported by the
> crypto device.
> 
> The create_session() function is refactored to use the
> create_sec_session() and create_crypto_session() functions.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call crypto_devid_fill()
> and max_session_size() and to call session_pool_init() and
> session_priv_pool_init().
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
>  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
>  examples/ipsec-secgw/ipsec.h         |  10 +-
>  examples/ipsec-secgw/ipsec_process.c |  38 +--
>  examples/ipsec-secgw/sa.c            |  42 ++-
>  5 files changed, 495 insertions(+), 435 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> secgw/ipsec-secgw.c
> index ffbd00b..cc8bb57 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -182,6 +182,14 @@ struct lcore_params {
>  	uint8_t lcore_id;
>  } __rte_cache_aligned;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> 
>  static struct lcore_params *lcore_params;
> @@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
>  	return -1;
>  }
> 
> +static void
> +crypto_devid_fill(void)
> +{
> +	uint32_t i, n;
> +
> +	n = rte_cryptodev_count();
> +
> +	for (i = 0; i != n; i++) {
> +		if (check_cryptodev_mask(i) == 0)
> +			crypto_devid[crypto_dev_num++] = i;
> +	}
> +}
> +
>  static int32_t
>  cryptodevs_init(void)
>  {
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
> +	uint32_t dev_max_sess;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1722,51 +1705,12 @@ cryptodevs_init(void)
>  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>  		dev_conf.nb_queue_pairs = qp;
> 
> -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> +		dev_max_sess = cdev_info.sym.max_nb_sessions;
>  		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
>  			rte_exit(EXIT_FAILURE,
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_pool = sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads,
> uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	crypto_devid_fill();
> +
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
> +
> +		if (!numa_on)
> +			break;
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 4352cb8..e31d472 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> rte_security_ipsec_xform *ipsec)
>  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
>  }
> 
> -int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +static int
> +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
>  {
> -	struct rte_cryptodev_info cdev_info;
> -	unsigned long cdev_id_qp = 0;
> +	const struct rte_security_capability *sec_cap;
> +	struct rte_security_ctx *ctx;
>  	int32_t ret = 0;
> -	struct cdev_key key = { 0 };
> -
> -	key.lcore_id = (uint8_t)rte_lcore_id();
> -
> -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> -	key.auth_algo = (uint8_t)sa->auth_algo;
> -	key.aead_algo = (uint8_t)sa->aead_algo;
> -
> -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> -				(void **)&cdev_id_qp);
> -		if (ret < 0) {
> -			RTE_LOG(ERR, IPSEC,
> -				"No cryptodev: core %u, cipher_algo %u, "
> -				"auth_algo %u, aead_algo %u\n",
> -				key.lcore_id,
> -				key.cipher_algo,
> -				key.auth_algo,
> -				key.aead_algo);
> -			return -1;
> -		}
> -	}
> 
> -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev
> "
> -			"%u qp %u\n", sa->spi,
> -			ipsec_ctx->tbl[cdev_id_qp].id,
> -			ipsec_ctx->tbl[cdev_id_qp].qp);
> +	if ((sa == NULL) || (pool == NULL))
> +		return -EINVAL;
> 
> -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -		struct rte_security_session_conf sess_conf = {
> +	struct rte_security_session_conf sess_conf = {
>  			.action_type = sa->type,
>  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>  			{.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>  			} },
>  			.crypto_xform = sa->xforms,
>  			.userdata = NULL,
> -
>  		};
> 
> -		if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_cryptodev_get_sec_ctx(
> -							ipsec_ctx-
> >tbl[cdev_id_qp].id);
> -
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_eth_dev_get_sec_ctx(
> -							sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -			int ret = 0;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> -					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +				"SEC Session init failed: err: %d\n",
> +				ret);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		struct rte_flow_error err;
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> +			return -1;
> +		}
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +		sec_cap = rte_security_capabilities_get(ctx);
> +
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
>  				"No suitable security capability found\n");
>  				return -1;
> -			}
> +		}
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> -
> -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> -			if (sa->flags & IP6_TUNNEL) {
> -				sa->pattern[1].spec = &sa->ipv6_spec;
> -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> -					sa->dst.ip.ip6.ip6_b, 16);
> -				memcpy(sa->ipv6_spec.hdr.src_addr,
> -				       sa->src.ip.ip6.ip6_b, 16);
> -			} else {
> -				sa->pattern[1].spec = &sa->ipv4_spec;
> -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> -			}
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> +
> +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> +		if (sa->flags & IP6_TUNNEL) {
> +			sa->pattern[1].spec = &sa->ipv6_spec;
> +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> +				sa->dst.ip.ip6.ip6_b, 16);
> +			memcpy(sa->ipv6_spec.hdr.src_addr,
> +			       sa->src.ip.ip6.ip6_b, 16);
> +		} else {
> +			sa->pattern[1].spec = &sa->ipv4_spec;
> +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> +		}
> 
> -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> -			sa->pattern[2].spec = &sa->esp_spec;
> -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> +		sa->pattern[2].spec = &sa->esp_spec;
> +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> 
> -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> 
> -			sa->action[0].type =
> RTE_FLOW_ACTION_TYPE_SECURITY;
> -			sa->action[0].conf = sa->sec_session;
> +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> +		sa->action[0].conf = sa->sec_session;
> 
> -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> 
> -			sa->attr.egress = (sa->direction ==
> +		sa->attr.egress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> -			sa->attr.ingress = (sa->direction ==
> +		sa->attr.ingress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> -			if (sa->attr.ingress) {
> -				uint8_t rss_key[40];
> -				struct rte_eth_rss_conf rss_conf = {
> -					.rss_key = rss_key,
> -					.rss_key_len = 40,
> -				};
> -				struct rte_eth_dev *eth_dev;
> -				uint16_t
> queue[RTE_MAX_QUEUES_PER_PORT];
> -				struct rte_flow_action_rss action_rss;
> -				unsigned int i;
> -				unsigned int j;
> -
> -				sa->action[2].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				/* Try RSS. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> -				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> -							      &rss_conf);
> -				for (i = 0, j = 0;
> -				     i < eth_dev->data->nb_rx_queues; ++i)
> -					if (eth_dev->data->rx_queues[i])
> -						queue[j++] = i;
> +		if (sa->attr.ingress) {
> +			uint8_t rss_key[40];
> +			struct rte_eth_rss_conf rss_conf = {
> +				.rss_key = rss_key,
> +				.rss_key_len = 40,
> +			};
> +			struct rte_eth_dev *eth_dev;
> +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> +			struct rte_flow_action_rss action_rss;
> +			unsigned int i;
> +			unsigned int j;
> +
> +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> +			/* Try RSS. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> +			sa->action[1].conf = &action_rss;
> +			eth_dev = ctx->device;
> +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> +						&rss_conf);
> +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> +				++i)
> +				if (eth_dev->data->rx_queues[i])
> +					queue[j++] = i;
Compilation error

/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In function 'create_sec_session':
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4: error: this 'for' clause does not guard... [-Werror=misleading-indentation]
    for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
    ^~~
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5: note: ...this statement, but the latter is misleadingly indented as if it is guarded by the 'for'
     action_rss = (struct rte_flow_action_rss){
     ^~~~~~~~~~

>  				action_rss = (struct rte_flow_action_rss){
>  					.types = rss_conf.rss_hf,
>  					.key_len = rss_conf.rss_key_len,
>  					.queue_num = j,
>  					.key = rss_key,
>  					.queue = queue,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				if (!ret)
> -					goto flow_create;
> -				/* Try Queue. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_QUEUE;
> -				sa->action[1].conf =
> -					&(struct rte_flow_action_queue){
> -					.index = 0,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				/* Try End. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				sa->action[1].conf = NULL;
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			if (!ret)
> +				goto flow_create;
> +			/* Try Queue. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
> +			sa->action[1].conf =
> +				&(struct rte_flow_action_queue){
> +				.index = 0,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			/* Try End. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +			sa->action[1].conf = NULL;
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
>  							sa->pattern, sa-
> >action,
>  							&err);
> -				if (ret)
> -					goto flow_create_failure;
> -			} else if (sa->attr.egress &&
> -				   (sa->ol_flags &
> -				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))
> {
> -				sa->action[1].type =
> -					RTE_FLOW_ACTION_TYPE_PASSTHRU;
> -				sa->action[2].type =
> -					RTE_FLOW_ACTION_TYPE_END;
> -			}
> +			if (ret)
> +				goto flow_create_failure;
> +		} else if (sa->attr.egress &&
> +				(sa->ol_flags &
> +				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
> +			sa->action[1].type =
> +				RTE_FLOW_ACTION_TYPE_PASSTHRU;
> +			sa->action[2].type =
> +				RTE_FLOW_ACTION_TYPE_END;
> +		}
>  flow_create:
> -			sa->flow = rte_flow_create(sa->portid,
> -				&sa->attr, sa->pattern, sa->action, &err);
> -			if (sa->flow == NULL) {
> +		sa->flow = rte_flow_create(sa->portid,
> +			&sa->attr, sa->pattern, sa->action, &err);
> +		if (sa->flow == NULL) {
>  flow_create_failure:
> -				RTE_LOG(ERR, IPSEC,
> -					"Failed to create ipsec flow msg: %s\n",
> -					err.message);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> -
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -
> -			if (ctx == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"Ethernet device doesn't have security features
> registered\n");
> -				return -1;
> -			}
> +			RTE_LOG(ERR, IPSEC,
> +				"Failed to create ipsec flow msg: %s\n",
> +				err.message);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> +		struct rte_security_ctx *ctx =
> +				(struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		const struct rte_security_capability *sec_cap;
> 
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			/* Save SA as userdata for the security session. When
> -			 * the packet is received, this userdata will be
> -			 * retrieved using the metadata from the packet.
> -			 *
> -			 * The PMD is expected to set similar metadata for
> other
> -			 * operations, like rte_eth_event, which are tied to
> -			 * security session. In such cases, the userdata could
> -			 * be obtained to uniquely identify the security
> -			 * parameters denoted.
> -			 */
> -
> -			sess_conf.userdata = (void *) sa;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +		if (ctx == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"Ethernet device doesn't have security features
> registered\n");
> +			return -1;
> +		}
> +
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> +
> +		/* Save SA as userdata for the security session. When
> +		 * the packet is received, this userdata will be
> +		 * retrieved using the metadata from the packet.
> +		 *
> +		 * The PMD is expected to set similar metadata for other
> +		 * operations, like rte_eth_event, which are tied to
> +		 * security session. In such cases, the userdata could
> +		 * be obtained to uniquely identify the security
> +		 * parameters denoted.
> +		 */
> +
> +		sess_conf.userdata = (void *) sa;
> +
> +		sa->sec_session = rte_security_session_create(ctx,
> +			&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"SEC Session init failed: err: %d\n", ret);
> +			return -1;
> +		}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		sec_cap = rte_security_capabilities_get(ctx);
> 
> -			if (sec_cap == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No capabilities registered\n");
> -				return -1;
> -			}
> +		if (sec_cap == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No capabilities registered\n");
> +			return -1;
> +		}
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action !=
>  					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No suitable security capability found\n");
> -				return -1;
> -			}
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No suitable security capability found\n");
> +			return -1;
> +		}
> +
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +	} else
> +		return -EINVAL;
> +	return 0;
> +}
> +
> +#define CDEV_IV_SIZE 12
> +
> +static int
> +check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
> +	cap_idx.algo.aead = ss->aead_algo;
> +
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap == NULL)
> +		return -ENOENT;
> +
> +	return rte_cryptodev_sym_capability_check_aead(cap,
> +					ss->cipher_key_len,
> +					ss->digest_len,
> +					ss->aad_len,
> +					CDEV_IV_SIZE);
> +}
> +
> +static int
> +check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> +	uint16_t auth_iv_len;
> +	int rc = -1;
> +
> +	if (ss == NULL)
> +		return rc;
> +
> +	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
> +		return check_cryptodev_aead_capablity(ss, dev_id);
> +
> +	auth_iv_len = 0;
> +
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
> +	cap_idx.algo.auth = ss->auth_algo;
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap != NULL) {
> +		rc = rte_cryptodev_sym_capability_check_auth(
> +				cap, ss->auth_key_len, ss->digest_len,
> +				auth_iv_len);
> +		if (rc == 0) {
> +			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +			cap_idx.algo.cipher = ss->cipher_algo;
> +			cap = rte_cryptodev_sym_capability_get(dev_id,
> +					&cap_idx);
> +			if (cap != NULL)
> +				rc =
> rte_cryptodev_sym_capability_check_cipher(
> +						cap, ss->cipher_key_len,
> +						ss->iv_len);
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
> -	sa->cdev_id_qp = cdev_id_qp;
> 
> -	return 0;
> +	return rc;
> +}
> +
> +static int
> +create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	int32_t rc;
> +	uint32_t devnum, i;
> +	struct rte_cryptodev_sym_session *s;
> +	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
> +
> +	/* check which cryptodevs support SA */
> +	devnum = 0;
> +	for (i = 0; i < crypto_dev_num; i++) {
> +		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
> +		if (rc == 0)
> +			devid[devnum++] = crypto_devid[i];
> +	}
> +
> +	if (devnum == 0)
> +		return -ENODEV;
> +
> +	s = rte_cryptodev_sym_session_create(pool);
> +	if (s == NULL)
> +		return -ENOMEM;
> +
> +	/* initialize SA crypto session for all supported devices */
> +	for (i = 0; i != devnum; i++) {
> +		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
> +			pool);
> +		if (rc != 0)
> +			break;
> +	}
> +
> +	if (i == devnum) {
> +		sa->crypto_session = s;
> +		return 0;
> +	}
> +
> +	/* failure, do cleanup */
> +	while (i-- != 0)
> +		rte_cryptodev_sym_session_clear(devid[i], s);
> +
> +	rte_cryptodev_sym_session_free(s);
> +	return rc;
> +}
> +
> +int
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
> +		return create_sec_session(sa, pool);
> +	else
> +		return create_crypto_session(sa, pool);
>  }
> 
>  /*
> @@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			sym_cop = get_sym_cop(&priv->cop);
>  			sym_cop->m_src = pkts[i];
> 
> @@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_crypto_op_attach_sym_session(&priv->cop,
>  					sa->crypto_session);
> 
> @@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index 99f49d6..804330c 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -83,6 +83,14 @@ struct app_sa_prm {
> 
>  extern struct app_sa_prm app_sa_prm;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +extern uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  struct ipsec_sa {
>  	struct rte_ipsec_session ips; /* one session per sa for now */
>  	uint32_t spi;
> @@ -306,6 +314,6 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-
> secgw/ipsec_process.c
> index 3f9cacb..0df6969 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct
> rte_crypto_op *cop[], uint32_t num)
>  	cqp->len = len;
>  }
> 
> -static inline int
> -fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
> -	struct ipsec_sa *sa)
> -{
> -	int32_t rc;
> -
> -	/* setup crypto section */
> -	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->crypto.ses = sa->crypto_session;
> -	/* setup session action type */
> -	} else {
> -		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->security.ses = sa->sec_session;
> -		ss->security.ctx = sa->security_ctx;
> -		ss->security.ol_flags = sa->ol_flags;
> -	}
> -
> -	rc = rte_ipsec_session_prepare(ss);
> -	if (rc != 0)
> -		memset(ss, 0, sizeof(*ss));
> -
> -	return rc;
> -}
> -
>  /*
>   * group input packets byt the SA they belong to.
>   */
> @@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic
> *trf)
> 
>  		ips = &sa->ips;
> 
> -		/* no valid HW session for that SA, try to create one */
> -		if (sa == NULL || (ips->crypto.ses == NULL &&
> -				fill_ipsec_session(ips, ctx, sa) != 0))
> +		/* no valid HW session for that SA */
> +		if (sa == NULL || ips->crypto.ses == NULL)
>  			k = 0;
> 
>  		/* process packets inline */
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index a7298a3..0f36f5b 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> inbound)
>  	return 0;
>  }
> 
> -
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  			print_one_sa_rule(sa, inbound);
>  		}
> +		rc = create_session(sa, skt_ctx->session_pool);
> +		if (rc != 0) {
> +			RTE_LOG(ERR, IPSEC_ESP,
> +					"create_session() failed\n");
> +			return -EINVAL;
> +		}
>  	}
> 
>  	return 0;
> @@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	rc = rte_ipsec_session_prepare(ss);
> +	if (rc != 0)
> +		memset(ss, 0, sizeof(*ss));
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa
> *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add inbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add outbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule.
> 
> Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
> sessions at startup to fix the issue of the first packet
> being dropped for inline crypto.
> 
> The create_session() function is now called at initialisation in
> sa_add_rules() which is called from sa_init().
> The return code for add_rules is checked.
> Calls to create_session() in other functions are dropped.
> 
> Add crypto_devid_fill() in ipsec-secgw.c
> Add max_session_size() in ipsec-secgw.c
> Add check_cryptodev_capability() in ipsec.c
> Add check_cryptodev_aead_capability() in ipsec.c
> Add create_sec_session() and create_crypto_session() in ipsec.c
> 
> The crypto_dev_fill() function has been added to find the
> enabled crypto devices.
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The check_cryptodev_capability() and check_cryptodev_aead_capability()
> functions have been added to check that the SA is supported by the
> crypto device.
> 
> The create_session() function is refactored to use the
> create_sec_session() and create_crypto_session() functions.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call crypto_devid_fill()
> and max_session_size() and to call session_pool_init() and
> session_priv_pool_init().
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
>  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++----------------
>  examples/ipsec-secgw/ipsec.h         |  10 +-
>  examples/ipsec-secgw/ipsec_process.c |  38 +--
>  examples/ipsec-secgw/sa.c            |  42 ++-
>  5 files changed, 495 insertions(+), 435 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> secgw/ipsec-secgw.c
> index ffbd00b..cc8bb57 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -182,6 +182,14 @@ struct lcore_params {
>  	uint8_t lcore_id;
>  } __rte_cache_aligned;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> 
>  static struct lcore_params *lcore_params;
> @@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
>  	return -1;
>  }
> 
> +static void
> +crypto_devid_fill(void)
> +{
> +	uint32_t i, n;
> +
> +	n = rte_cryptodev_count();
> +
> +	for (i = 0; i != n; i++) {
> +		if (check_cryptodev_mask(i) == 0)
> +			crypto_devid[crypto_dev_num++] = i;
> +	}
> +}
> +
>  static int32_t
>  cryptodevs_init(void)
>  {
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
> +	uint32_t dev_max_sess;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1722,51 +1705,12 @@ cryptodevs_init(void)
>  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>  		dev_conf.nb_queue_pairs = qp;
> 
> -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> +		dev_max_sess = cdev_info.sym.max_nb_sessions;
>  		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
>  			rte_exit(EXIT_FAILURE,
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_pool = sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads,
> uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	crypto_devid_fill();
> +
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
> +
> +		if (!numa_on)
> +			break;
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 4352cb8..e31d472 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> rte_security_ipsec_xform *ipsec)
>  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
>  }
> 
> -int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +static int
> +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
>  {
> -	struct rte_cryptodev_info cdev_info;
> -	unsigned long cdev_id_qp = 0;
> +	const struct rte_security_capability *sec_cap;
> +	struct rte_security_ctx *ctx;
>  	int32_t ret = 0;
> -	struct cdev_key key = { 0 };
> -
> -	key.lcore_id = (uint8_t)rte_lcore_id();
> -
> -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> -	key.auth_algo = (uint8_t)sa->auth_algo;
> -	key.aead_algo = (uint8_t)sa->aead_algo;
> -
> -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> -				(void **)&cdev_id_qp);
> -		if (ret < 0) {
> -			RTE_LOG(ERR, IPSEC,
> -				"No cryptodev: core %u, cipher_algo %u, "
> -				"auth_algo %u, aead_algo %u\n",
> -				key.lcore_id,
> -				key.cipher_algo,
> -				key.auth_algo,
> -				key.aead_algo);
> -			return -1;
> -		}
> -	}
> 
> -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev
> "
> -			"%u qp %u\n", sa->spi,
> -			ipsec_ctx->tbl[cdev_id_qp].id,
> -			ipsec_ctx->tbl[cdev_id_qp].qp);
> +	if ((sa == NULL) || (pool == NULL))
> +		return -EINVAL;
> 
> -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> -		struct rte_security_session_conf sess_conf = {
> +	struct rte_security_session_conf sess_conf = {
>  			.action_type = sa->type,
>  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
>  			{.ipsec = {
> @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
>  			} },
>  			.crypto_xform = sa->xforms,
>  			.userdata = NULL,
> -
>  		};
> 
> -		if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_cryptodev_get_sec_ctx(
> -							ipsec_ctx-
> >tbl[cdev_id_qp].id);
> -
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> -			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -
> 	rte_eth_dev_get_sec_ctx(
> -							sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -			int ret = 0;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> -					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +				"SEC Session init failed: err: %d\n",
> +				ret);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		struct rte_flow_error err;
> +		ctx = (struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		sa->sec_session = rte_security_session_create(ctx,
> +				&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> +			return -1;
> +		}
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +		sec_cap = rte_security_capabilities_get(ctx);
> +
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE)
> {
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
>  				"No suitable security capability found\n");
>  				return -1;
> -			}
> +		}
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> -
> -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> -			if (sa->flags & IP6_TUNNEL) {
> -				sa->pattern[1].spec = &sa->ipv6_spec;
> -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> -					sa->dst.ip.ip6.ip6_b, 16);
> -				memcpy(sa->ipv6_spec.hdr.src_addr,
> -				       sa->src.ip.ip6.ip6_b, 16);
> -			} else {
> -				sa->pattern[1].spec = &sa->ipv4_spec;
> -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> -			}
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> +
> +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> +		if (sa->flags & IP6_TUNNEL) {
> +			sa->pattern[1].spec = &sa->ipv6_spec;
> +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> +				sa->dst.ip.ip6.ip6_b, 16);
> +			memcpy(sa->ipv6_spec.hdr.src_addr,
> +			       sa->src.ip.ip6.ip6_b, 16);
> +		} else {
> +			sa->pattern[1].spec = &sa->ipv4_spec;
> +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> +		}
> 
> -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> -			sa->pattern[2].spec = &sa->esp_spec;
> -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> +		sa->pattern[2].spec = &sa->esp_spec;
> +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> 
> -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> 
> -			sa->action[0].type =
> RTE_FLOW_ACTION_TYPE_SECURITY;
> -			sa->action[0].conf = sa->sec_session;
> +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> +		sa->action[0].conf = sa->sec_session;
> 
> -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> 
> -			sa->attr.egress = (sa->direction ==
> +		sa->attr.egress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> -			sa->attr.ingress = (sa->direction ==
> +		sa->attr.ingress = (sa->direction ==
> 
> 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> -			if (sa->attr.ingress) {
> -				uint8_t rss_key[40];
> -				struct rte_eth_rss_conf rss_conf = {
> -					.rss_key = rss_key,
> -					.rss_key_len = 40,
> -				};
> -				struct rte_eth_dev *eth_dev;
> -				uint16_t
> queue[RTE_MAX_QUEUES_PER_PORT];
> -				struct rte_flow_action_rss action_rss;
> -				unsigned int i;
> -				unsigned int j;
> -
> -				sa->action[2].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				/* Try RSS. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> -				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> -							      &rss_conf);
> -				for (i = 0, j = 0;
> -				     i < eth_dev->data->nb_rx_queues; ++i)
> -					if (eth_dev->data->rx_queues[i])
> -						queue[j++] = i;
> +		if (sa->attr.ingress) {
> +			uint8_t rss_key[40];
> +			struct rte_eth_rss_conf rss_conf = {
> +				.rss_key = rss_key,
> +				.rss_key_len = 40,
> +			};
> +			struct rte_eth_dev *eth_dev;
> +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> +			struct rte_flow_action_rss action_rss;
> +			unsigned int i;
> +			unsigned int j;
> +
> +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> +			/* Try RSS. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> +			sa->action[1].conf = &action_rss;
> +			eth_dev = ctx->device;
> +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> +						&rss_conf);
> +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> +				++i)
> +				if (eth_dev->data->rx_queues[i])
> +					queue[j++] = i;
Compilation error

/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In function 'create_sec_session':
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4: error: this 'for' clause does not guard... [-Werror=misleading-indentation]
    for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
    ^~~
/home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5: note: ...this statement, but the latter is misleadingly indented as if it is guarded by the 'for'
     action_rss = (struct rte_flow_action_rss){
     ^~~~~~~~~~

>  				action_rss = (struct rte_flow_action_rss){
>  					.types = rss_conf.rss_hf,
>  					.key_len = rss_conf.rss_key_len,
>  					.queue_num = j,
>  					.key = rss_key,
>  					.queue = queue,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				if (!ret)
> -					goto flow_create;
> -				/* Try Queue. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_QUEUE;
> -				sa->action[1].conf =
> -					&(struct rte_flow_action_queue){
> -					.index = 0,
> -				};
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> -							sa->pattern, sa-
> >action,
> -							&err);
> -				/* Try End. */
> -				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_END;
> -				sa->action[1].conf = NULL;
> -				ret = rte_flow_validate(sa->portid, &sa->attr,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			if (!ret)
> +				goto flow_create;
> +			/* Try Queue. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
> +			sa->action[1].conf =
> +				&(struct rte_flow_action_queue){
> +				.index = 0,
> +			};
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
> +						sa->pattern, sa->action,
> +						&err);
> +			/* Try End. */
> +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> +			sa->action[1].conf = NULL;
> +			ret = rte_flow_validate(sa->portid, &sa->attr,
>  							sa->pattern, sa-
> >action,
>  							&err);
> -				if (ret)
> -					goto flow_create_failure;
> -			} else if (sa->attr.egress &&
> -				   (sa->ol_flags &
> -				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD))
> {
> -				sa->action[1].type =
> -					RTE_FLOW_ACTION_TYPE_PASSTHRU;
> -				sa->action[2].type =
> -					RTE_FLOW_ACTION_TYPE_END;
> -			}
> +			if (ret)
> +				goto flow_create_failure;
> +		} else if (sa->attr.egress &&
> +				(sa->ol_flags &
> +				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
> +			sa->action[1].type =
> +				RTE_FLOW_ACTION_TYPE_PASSTHRU;
> +			sa->action[2].type =
> +				RTE_FLOW_ACTION_TYPE_END;
> +		}
>  flow_create:
> -			sa->flow = rte_flow_create(sa->portid,
> -				&sa->attr, sa->pattern, sa->action, &err);
> -			if (sa->flow == NULL) {
> +		sa->flow = rte_flow_create(sa->portid,
> +			&sa->attr, sa->pattern, sa->action, &err);
> +		if (sa->flow == NULL) {
>  flow_create_failure:
> -				RTE_LOG(ERR, IPSEC,
> -					"Failed to create ipsec flow msg: %s\n",
> -					err.message);
> -				return -1;
> -			}
> -		} else if (sa->type ==
> -
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
> -			const struct rte_security_capability *sec_cap;
> -
> -			if (ctx == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"Ethernet device doesn't have security features
> registered\n");
> -				return -1;
> -			}
> +			RTE_LOG(ERR, IPSEC,
> +				"Failed to create ipsec flow msg: %s\n",
> +				err.message);
> +			return -1;
> +		}
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> +		struct rte_security_ctx *ctx =
> +				(struct rte_security_ctx *)
> +				rte_eth_dev_get_sec_ctx(sa->portid);
> +		const struct rte_security_capability *sec_cap;
> 
> -			/* Set IPsec parameters in conf */
> -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> -
> -			/* Save SA as userdata for the security session. When
> -			 * the packet is received, this userdata will be
> -			 * retrieved using the metadata from the packet.
> -			 *
> -			 * The PMD is expected to set similar metadata for
> other
> -			 * operations, like rte_eth_event, which are tied to
> -			 * security session. In such cases, the userdata could
> -			 * be obtained to uniquely identify the security
> -			 * parameters denoted.
> -			 */
> -
> -			sess_conf.userdata = (void *) sa;
> -
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> -			if (sa->sec_session == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"SEC Session init failed: err: %d\n", ret);
> -				return -1;
> -			}
> +		if (ctx == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"Ethernet device doesn't have security features
> registered\n");
> +			return -1;
> +		}
> +
> +		/* Set IPsec parameters in conf */
> +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> +
> +		/* Save SA as userdata for the security session. When
> +		 * the packet is received, this userdata will be
> +		 * retrieved using the metadata from the packet.
> +		 *
> +		 * The PMD is expected to set similar metadata for other
> +		 * operations, like rte_eth_event, which are tied to
> +		 * security session. In such cases, the userdata could
> +		 * be obtained to uniquely identify the security
> +		 * parameters denoted.
> +		 */
> +
> +		sess_conf.userdata = (void *) sa;
> +
> +		sa->sec_session = rte_security_session_create(ctx,
> +			&sess_conf, pool);
> +		if (sa->sec_session == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"SEC Session init failed: err: %d\n", ret);
> +			return -1;
> +		}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +		sec_cap = rte_security_capabilities_get(ctx);
> 
> -			if (sec_cap == NULL) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No capabilities registered\n");
> -				return -1;
> -			}
> +		if (sec_cap == NULL) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No capabilities registered\n");
> +			return -1;
> +		}
> 
> -			/* iterate until ESP tunnel*/
> -			while (sec_cap->action !=
> +		/* iterate until ESP tunnel*/
> +		while (sec_cap->action !=
>  					RTE_SECURITY_ACTION_TYPE_NONE)
> {
> 
> -				if (sec_cap->action == sa->type &&
> -				    sec_cap->protocol ==
> -					RTE_SECURITY_PROTOCOL_IPSEC &&
> -				    sec_cap->ipsec.mode ==
> -
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> -				    sec_cap->ipsec.direction == sa->direction)
> -					break;
> -				sec_cap++;
> -			}
> +			if (sec_cap->action == sa->type &&
> +				sec_cap->protocol ==
> +				RTE_SECURITY_PROTOCOL_IPSEC &&
> +				sec_cap->ipsec.mode ==
> +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> +				sec_cap->ipsec.direction == sa->direction)
> +				break;
> +			sec_cap++;
> +		}
> 
> -			if (sec_cap->action ==
> RTE_SECURITY_ACTION_TYPE_NONE) {
> -				RTE_LOG(ERR, IPSEC,
> -				"No suitable security capability found\n");
> -				return -1;
> -			}
> +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
> +			RTE_LOG(ERR, IPSEC,
> +			"No suitable security capability found\n");
> +			return -1;
> +		}
> +
> +		sa->ol_flags = sec_cap->ol_flags;
> +		sa->security_ctx = ctx;
> +	} else
> +		return -EINVAL;
> +	return 0;
> +}
> +
> +#define CDEV_IV_SIZE 12
> +
> +static int
> +check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> 
> -			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
> +	cap_idx.algo.aead = ss->aead_algo;
> +
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap == NULL)
> +		return -ENOENT;
> +
> +	return rte_cryptodev_sym_capability_check_aead(cap,
> +					ss->cipher_key_len,
> +					ss->digest_len,
> +					ss->aad_len,
> +					CDEV_IV_SIZE);
> +}
> +
> +static int
> +check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
> +{
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	const struct rte_cryptodev_symmetric_capability *cap;
> +	uint16_t auth_iv_len;
> +	int rc = -1;
> +
> +	if (ss == NULL)
> +		return rc;
> +
> +	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
> +		return check_cryptodev_aead_capablity(ss, dev_id);
> +
> +	auth_iv_len = 0;
> +
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
> +	cap_idx.algo.auth = ss->auth_algo;
> +	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
> +	if (cap != NULL) {
> +		rc = rte_cryptodev_sym_capability_check_auth(
> +				cap, ss->auth_key_len, ss->digest_len,
> +				auth_iv_len);
> +		if (rc == 0) {
> +			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +			cap_idx.algo.cipher = ss->cipher_algo;
> +			cap = rte_cryptodev_sym_capability_get(dev_id,
> +					&cap_idx);
> +			if (cap != NULL)
> +				rc =
> rte_cryptodev_sym_capability_check_cipher(
> +						cap, ss->cipher_key_len,
> +						ss->iv_len);
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
> -	sa->cdev_id_qp = cdev_id_qp;
> 
> -	return 0;
> +	return rc;
> +}
> +
> +static int
> +create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	int32_t rc;
> +	uint32_t devnum, i;
> +	struct rte_cryptodev_sym_session *s;
> +	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
> +
> +	/* check which cryptodevs support SA */
> +	devnum = 0;
> +	for (i = 0; i < crypto_dev_num; i++) {
> +		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
> +		if (rc == 0)
> +			devid[devnum++] = crypto_devid[i];
> +	}
> +
> +	if (devnum == 0)
> +		return -ENODEV;
> +
> +	s = rte_cryptodev_sym_session_create(pool);
> +	if (s == NULL)
> +		return -ENOMEM;
> +
> +	/* initialize SA crypto session for all supported devices */
> +	for (i = 0; i != devnum; i++) {
> +		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
> +			pool);
> +		if (rc != 0)
> +			break;
> +	}
> +
> +	if (i == devnum) {
> +		sa->crypto_session = s;
> +		return 0;
> +	}
> +
> +	/* failure, do cleanup */
> +	while (i-- != 0)
> +		rte_cryptodev_sym_session_clear(devid[i], s);
> +
> +	rte_cryptodev_sym_session_free(s);
> +	return rc;
> +}
> +
> +int
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> +{
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
> +		return create_sec_session(sa, pool);
> +	else
> +		return create_crypto_session(sa, pool);
>  }
> 
>  /*
> @@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			sym_cop = get_sym_cop(&priv->cop);
>  			sym_cop->m_src = pkts[i];
> 
> @@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_crypto_op_attach_sym_session(&priv->cop,
>  					sa->crypto_session);
> 
> @@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index 99f49d6..804330c 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -83,6 +83,14 @@ struct app_sa_prm {
> 
>  extern struct app_sa_prm app_sa_prm;
> 
> +/*
> + * Number of enabled crypto devices
> + * This number is needed when checking crypto device capabilities
> + */
> +extern uint8_t crypto_dev_num;
> +/* array of crypto device ID's */
> +extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> +
>  struct ipsec_sa {
>  	struct rte_ipsec_session ips; /* one session per sa for now */
>  	uint32_t spi;
> @@ -306,6 +314,6 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-
> secgw/ipsec_process.c
> index 3f9cacb..0df6969 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct
> rte_crypto_op *cop[], uint32_t num)
>  	cqp->len = len;
>  }
> 
> -static inline int
> -fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
> -	struct ipsec_sa *sa)
> -{
> -	int32_t rc;
> -
> -	/* setup crypto section */
> -	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> -		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->crypto.ses = sa->crypto_session;
> -	/* setup session action type */
> -	} else {
> -		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> -			if (rc != 0)
> -				return rc;
> -		}
> -		ss->security.ses = sa->sec_session;
> -		ss->security.ctx = sa->security_ctx;
> -		ss->security.ol_flags = sa->ol_flags;
> -	}
> -
> -	rc = rte_ipsec_session_prepare(ss);
> -	if (rc != 0)
> -		memset(ss, 0, sizeof(*ss));
> -
> -	return rc;
> -}
> -
>  /*
>   * group input packets byt the SA they belong to.
>   */
> @@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic
> *trf)
> 
>  		ips = &sa->ips;
> 
> -		/* no valid HW session for that SA, try to create one */
> -		if (sa == NULL || (ips->crypto.ses == NULL &&
> -				fill_ipsec_session(ips, ctx, sa) != 0))
> +		/* no valid HW session for that SA */
> +		if (sa == NULL || ips->crypto.ses == NULL)
>  			k = 0;
> 
>  		/* process packets inline */
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index a7298a3..0f36f5b 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> inbound)
>  	return 0;
>  }
> 
> -
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  			print_one_sa_rule(sa, inbound);
>  		}
> +		rc = create_session(sa, skt_ctx->session_pool);
> +		if (rc != 0) {
> +			RTE_LOG(ERR, IPSEC_ESP,
> +					"create_session() failed\n");
> +			return -EINVAL;
> +		}
>  	}
> 
>  	return 0;
> @@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	rc = rte_ipsec_session_prepare(ss);
> +	if (rc != 0)
> +		memset(ss, 0, sizeof(*ss));
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa
> *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add inbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> +		if (rc != 0)
> +			rte_exit(EXIT_FAILURE,
> +				"failed to add outbound rules\n");
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, April 17, 2019 12:52 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
> 
> >
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule.
> >
> > Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create sessions
> > at startup to fix the issue of the first packet being dropped for
> > inline crypto.
> >
> > The create_session() function is now called at initialisation in
> > sa_add_rules() which is called from sa_init().
> > The return code for add_rules is checked.
> > Calls to create_session() in other functions are dropped.
> >
> > Add crypto_devid_fill() in ipsec-secgw.c Add max_session_size() in
> > ipsec-secgw.c Add check_cryptodev_capability() in ipsec.c Add
> > check_cryptodev_aead_capability() in ipsec.c Add create_sec_session()
> > and create_crypto_session() in ipsec.c
> >
> > The crypto_dev_fill() function has been added to find the enabled
> > crypto devices.
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The check_cryptodev_capability() and check_cryptodev_aead_capability()
> > functions have been added to check that the SA is supported by the
> > crypto device.
> >
> > The create_session() function is refactored to use the
> > create_sec_session() and create_crypto_session() functions.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call crypto_devid_fill()
> > and max_session_size() and to call session_pool_init() and
> > session_priv_pool_init().
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
> >  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++-----------
> -----
> >  examples/ipsec-secgw/ipsec.h         |  10 +-
> >  examples/ipsec-secgw/ipsec_process.c |  38 +--
> >  examples/ipsec-secgw/sa.c            |  42 ++-
> >  5 files changed, 495 insertions(+), 435 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> > secgw/ipsec-secgw.c index ffbd00b..cc8bb57 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -182,6 +182,14 @@ struct lcore_params {
> >  	uint8_t lcore_id;
> >  } __rte_cache_aligned;
> >
> > +/*
> > + * Number of enabled crypto devices
> > + * This number is needed when checking crypto device capabilities  */
> > +uint8_t crypto_dev_num;
> > +/* array of crypto device ID's */
> > +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> > +
> >  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> >
> >  static struct lcore_params *lcore_params; @@ -1623,13 +1631,27 @@
> > check_cryptodev_mask(uint8_t cdev_id)
> >  	return -1;
> >  }
> >
> > +static void
> > +crypto_devid_fill(void)
> > +{
> > +	uint32_t i, n;
> > +
> > +	n = rte_cryptodev_count();
> > +
> > +	for (i = 0; i != n; i++) {
> > +		if (check_cryptodev_mask(i) == 0)
> > +			crypto_devid[crypto_dev_num++] = i;
> > +	}
> > +}
> > +
> >  static int32_t
> >  cryptodevs_init(void)
> >  {
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> > +	uint32_t dev_max_sess;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1722,51 +1705,12
> @@
> > cryptodevs_init(void)
> >  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
> >  		dev_conf.nb_queue_pairs = qp;
> >
> > -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> > +		dev_max_sess = cdev_info.sym.max_nb_sessions;
> >  		if (dev_max_sess != 0 && dev_max_sess <
> CDEV_MP_NB_OBJS)
> >  			rte_exit(EXIT_FAILURE,
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp =
> rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> > sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> > dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -
> 	!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_pool =
> sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t
> > req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n",
> socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",
> 	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	crypto_devid_fill();
> > +
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id,
> sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> > +
> > +		if (!numa_on)
> > +			break;
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 4352cb8..e31d472 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)
> >  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;  }
> >
> > -int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +static int
> > +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> >  {
> > -	struct rte_cryptodev_info cdev_info;
> > -	unsigned long cdev_id_qp = 0;
> > +	const struct rte_security_capability *sec_cap;
> > +	struct rte_security_ctx *ctx;
> >  	int32_t ret = 0;
> > -	struct cdev_key key = { 0 };
> > -
> > -	key.lcore_id = (uint8_t)rte_lcore_id();
> > -
> > -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> > -	key.auth_algo = (uint8_t)sa->auth_algo;
> > -	key.aead_algo = (uint8_t)sa->aead_algo;
> > -
> > -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> > -				(void **)&cdev_id_qp);
> > -		if (ret < 0) {
> > -			RTE_LOG(ERR, IPSEC,
> > -				"No cryptodev: core %u, cipher_algo %u, "
> > -				"auth_algo %u, aead_algo %u\n",
> > -				key.lcore_id,
> > -				key.cipher_algo,
> > -				key.auth_algo,
> > -				key.aead_algo);
> > -			return -1;
> > -		}
> > -	}
> >
> > -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev
> > "
> > -			"%u qp %u\n", sa->spi,
> > -			ipsec_ctx->tbl[cdev_id_qp].id,
> > -			ipsec_ctx->tbl[cdev_id_qp].qp);
> > +	if ((sa == NULL) || (pool == NULL))
> > +		return -EINVAL;
> >
> > -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		struct rte_security_session_conf sess_conf = {
> > +	struct rte_security_session_conf sess_conf = {
> >  			.action_type = sa->type,
> >  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >  			{.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >  			} },
> >  			.crypto_xform = sa->xforms,
> >  			.userdata = NULL,
> > -
> >  		};
> >
> > -		if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_cryptodev_get_sec_ctx(
> > -							ipsec_ctx-
> > >tbl[cdev_id_qp].id);
> > -
> > -			/* Set IPsec parameters in conf */
> > -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > -		} else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > -			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> > -			const struct rte_security_capability *sec_cap;
> > -			int ret = 0;
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > +	if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +		/* Set IPsec parameters in conf */
> > +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -			/* iterate until ESP tunnel*/
> > -			while (sec_cap->action !=
> > -
> 	RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC,
> > +				"SEC Session init failed: err: %d\n",
> > +				ret);
> > +			return -1;
> > +		}
> > +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > +		struct rte_flow_error err;
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> > +			return -1;
> > +		}
> >
> > -				if (sec_cap->action == sa->type &&
> > -				    sec_cap->protocol ==
> > -					RTE_SECURITY_PROTOCOL_IPSEC &&
> > -				    sec_cap->ipsec.mode ==
> > -
> > 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> > -				    sec_cap->ipsec.direction == sa->direction)
> > -					break;
> > -				sec_cap++;
> > -			}
> > +		sec_cap = rte_security_capabilities_get(ctx);
> > +
> > +		/* iterate until ESP tunnel*/
> > +		while (sec_cap->action !=
> RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +			if (sec_cap->action == sa->type &&
> > +				sec_cap->protocol ==
> > +				RTE_SECURITY_PROTOCOL_IPSEC &&
> > +				sec_cap->ipsec.mode ==
> > +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
> &&
> > +				sec_cap->ipsec.direction == sa->direction)
> > +				break;
> > +			sec_cap++;
> > +		}
> >
> > -			if (sec_cap->action ==
> > RTE_SECURITY_ACTION_TYPE_NONE) {
> > -				RTE_LOG(ERR, IPSEC,
> > +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE)
> {
> > +			RTE_LOG(ERR, IPSEC,
> >  				"No suitable security capability found\n");
> >  				return -1;
> > -			}
> > +		}
> >
> > -			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > -
> > -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > -			if (sa->flags & IP6_TUNNEL) {
> > -				sa->pattern[1].spec = &sa->ipv6_spec;
> > -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> > -					sa->dst.ip.ip6.ip6_b, 16);
> > -				memcpy(sa->ipv6_spec.hdr.src_addr,
> > -				       sa->src.ip.ip6.ip6_b, 16);
> > -			} else {
> > -				sa->pattern[1].spec = &sa->ipv4_spec;
> > -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > -			}
> > +		sa->ol_flags = sec_cap->ol_flags;
> > +		sa->security_ctx = ctx;
> > +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > +
> > +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > +		if (sa->flags & IP6_TUNNEL) {
> > +			sa->pattern[1].spec = &sa->ipv6_spec;
> > +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> > +				sa->dst.ip.ip6.ip6_b, 16);
> > +			memcpy(sa->ipv6_spec.hdr.src_addr,
> > +			       sa->src.ip.ip6.ip6_b, 16);
> > +		} else {
> > +			sa->pattern[1].spec = &sa->ipv4_spec;
> > +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > +		}
> >
> > -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > -			sa->pattern[2].spec = &sa->esp_spec;
> > -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> > +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > +		sa->pattern[2].spec = &sa->esp_spec;
> > +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> >
> > -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> > +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> >
> > -			sa->action[0].type =
> > RTE_FLOW_ACTION_TYPE_SECURITY;
> > -			sa->action[0].conf = sa->sec_session;
> > +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> > +		sa->action[0].conf = sa->sec_session;
> >
> > -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> > +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> >
> > -			sa->attr.egress = (sa->direction ==
> > +		sa->attr.egress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> > -			sa->attr.ingress = (sa->direction ==
> > +		sa->attr.ingress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> > -			if (sa->attr.ingress) {
> > -				uint8_t rss_key[40];
> > -				struct rte_eth_rss_conf rss_conf = {
> > -					.rss_key = rss_key,
> > -					.rss_key_len = 40,
> > -				};
> > -				struct rte_eth_dev *eth_dev;
> > -				uint16_t
> > queue[RTE_MAX_QUEUES_PER_PORT];
> > -				struct rte_flow_action_rss action_rss;
> > -				unsigned int i;
> > -				unsigned int j;
> > -
> > -				sa->action[2].type =
> > RTE_FLOW_ACTION_TYPE_END;
> > -				/* Try RSS. */
> > -				sa->action[1].type =
> > RTE_FLOW_ACTION_TYPE_RSS;
> > -				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> > -							      &rss_conf);
> > -				for (i = 0, j = 0;
> > -				     i < eth_dev->data->nb_rx_queues; ++i)
> > -					if (eth_dev->data->rx_queues[i])
> > -						queue[j++] = i;
> > +		if (sa->attr.ingress) {
> > +			uint8_t rss_key[40];
> > +			struct rte_eth_rss_conf rss_conf = {
> > +				.rss_key = rss_key,
> > +				.rss_key_len = 40,
> > +			};
> > +			struct rte_eth_dev *eth_dev;
> > +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> > +			struct rte_flow_action_rss action_rss;
> > +			unsigned int i;
> > +			unsigned int j;
> > +
> > +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> > +			/* Try RSS. */
> > +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> > +			sa->action[1].conf = &action_rss;
> > +			eth_dev = ctx->device;
> > +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> > +						&rss_conf);
> > +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > +				++i)
> > +				if (eth_dev->data->rx_queues[i])
> > +					queue[j++] = i;
> Compilation error
> 
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> function 'create_sec_session':
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> error: this 'for' clause does not guard... [-Werror=misleading-indentation]
>     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
>     ^~~
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> note: ...this statement, but the latter is misleadingly indented as if it is
> guarded by the 'for'
>      action_rss = (struct rte_flow_action_rss){
>      ^~~~~~~~~~
> 
<snip>

I will send a v4.
What compiler are you using?

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, April 17, 2019 12:52 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped
> for inline crypto
> 
> >
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule.
> >
> > Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create sessions
> > at startup to fix the issue of the first packet being dropped for
> > inline crypto.
> >
> > The create_session() function is now called at initialisation in
> > sa_add_rules() which is called from sa_init().
> > The return code for add_rules is checked.
> > Calls to create_session() in other functions are dropped.
> >
> > Add crypto_devid_fill() in ipsec-secgw.c Add max_session_size() in
> > ipsec-secgw.c Add check_cryptodev_capability() in ipsec.c Add
> > check_cryptodev_aead_capability() in ipsec.c Add create_sec_session()
> > and create_crypto_session() in ipsec.c
> >
> > The crypto_dev_fill() function has been added to find the enabled
> > crypto devices.
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The check_cryptodev_capability() and check_cryptodev_aead_capability()
> > functions have been added to check that the SA is supported by the
> > crypto device.
> >
> > The create_session() function is refactored to use the
> > create_sec_session() and create_crypto_session() functions.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call crypto_devid_fill()
> > and max_session_size() and to call session_pool_init() and
> > session_priv_pool_init().
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 271 +++++++++--------
> >  examples/ipsec-secgw/ipsec.c         | 569 +++++++++++++++++++-----------
> -----
> >  examples/ipsec-secgw/ipsec.h         |  10 +-
> >  examples/ipsec-secgw/ipsec_process.c |  38 +--
> >  examples/ipsec-secgw/sa.c            |  42 ++-
> >  5 files changed, 495 insertions(+), 435 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
> > secgw/ipsec-secgw.c index ffbd00b..cc8bb57 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -182,6 +182,14 @@ struct lcore_params {
> >  	uint8_t lcore_id;
> >  } __rte_cache_aligned;
> >
> > +/*
> > + * Number of enabled crypto devices
> > + * This number is needed when checking crypto device capabilities  */
> > +uint8_t crypto_dev_num;
> > +/* array of crypto device ID's */
> > +uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
> > +
> >  static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
> >
> >  static struct lcore_params *lcore_params; @@ -1623,13 +1631,27 @@
> > check_cryptodev_mask(uint8_t cdev_id)
> >  	return -1;
> >  }
> >
> > +static void
> > +crypto_devid_fill(void)
> > +{
> > +	uint32_t i, n;
> > +
> > +	n = rte_cryptodev_count();
> > +
> > +	for (i = 0; i != n; i++) {
> > +		if (check_cryptodev_mask(i) == 0)
> > +			crypto_devid[crypto_dev_num++] = i;
> > +	}
> > +}
> > +
> >  static int32_t
> >  cryptodevs_init(void)
> >  {
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> > +	uint32_t dev_max_sess;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1652,45 +1674,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1722,51 +1705,12
> @@
> > cryptodevs_init(void)
> >  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
> >  		dev_conf.nb_queue_pairs = qp;
> >
> > -		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
> > +		dev_max_sess = cdev_info.sym.max_nb_sessions;
> >  		if (dev_max_sess != 0 && dev_max_sess <
> CDEV_MP_NB_OBJS)
> >  			rte_exit(EXIT_FAILURE,
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp =
> rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> > sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> > dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -
> 	!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1787,38 +1731,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_pool =
> sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t
> > req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n",
> socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",
> 	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	crypto_devid_fill();
> > +
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id,
> sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> > +
> > +		if (!numa_on)
> > +			break;
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 4352cb8..e31d472 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)
> >  	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;  }
> >
> > -int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +static int
> > +create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
> >  {
> > -	struct rte_cryptodev_info cdev_info;
> > -	unsigned long cdev_id_qp = 0;
> > +	const struct rte_security_capability *sec_cap;
> > +	struct rte_security_ctx *ctx;
> >  	int32_t ret = 0;
> > -	struct cdev_key key = { 0 };
> > -
> > -	key.lcore_id = (uint8_t)rte_lcore_id();
> > -
> > -	key.cipher_algo = (uint8_t)sa->cipher_algo;
> > -	key.auth_algo = (uint8_t)sa->auth_algo;
> > -	key.aead_algo = (uint8_t)sa->aead_algo;
> > -
> > -	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
> > -				(void **)&cdev_id_qp);
> > -		if (ret < 0) {
> > -			RTE_LOG(ERR, IPSEC,
> > -				"No cryptodev: core %u, cipher_algo %u, "
> > -				"auth_algo %u, aead_algo %u\n",
> > -				key.lcore_id,
> > -				key.cipher_algo,
> > -				key.auth_algo,
> > -				key.aead_algo);
> > -			return -1;
> > -		}
> > -	}
> >
> > -	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on
> cryptodev
> > "
> > -			"%u qp %u\n", sa->spi,
> > -			ipsec_ctx->tbl[cdev_id_qp].id,
> > -			ipsec_ctx->tbl[cdev_id_qp].qp);
> > +	if ((sa == NULL) || (pool == NULL))
> > +		return -EINVAL;
> >
> > -	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> > -		struct rte_security_session_conf sess_conf = {
> > +	struct rte_security_session_conf sess_conf = {
> >  			.action_type = sa->type,
> >  			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> >  			{.ipsec = {
> > @@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >  			} },
> >  			.crypto_xform = sa->xforms,
> >  			.userdata = NULL,
> > -
> >  		};
> >
> > -		if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_cryptodev_get_sec_ctx(
> > -							ipsec_ctx-
> > >tbl[cdev_id_qp].id);
> > -
> > -			/* Set IPsec parameters in conf */
> > -			set_ipsec_conf(sa, &(sess_conf.ipsec));
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > -		} else if (sa->type ==
> > RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > -			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx
> *)
> > -
> > 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> > -			const struct rte_security_capability *sec_cap;
> > -			int ret = 0;
> > -
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_pool);
> > -			if (sa->sec_session == NULL) {
> > -				RTE_LOG(ERR, IPSEC,
> > -				"SEC Session init failed: err: %d\n", ret);
> > -				return -1;
> > -			}
> > +	if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +		/* Set IPsec parameters in conf */
> > +		set_ipsec_conf(sa, &(sess_conf.ipsec));
> >
> > -			/* iterate until ESP tunnel*/
> > -			while (sec_cap->action !=
> > -
> 	RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC,
> > +				"SEC Session init failed: err: %d\n",
> > +				ret);
> > +			return -1;
> > +		}
> > +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> > +		struct rte_flow_error err;
> > +		ctx = (struct rte_security_ctx *)
> > +				rte_eth_dev_get_sec_ctx(sa->portid);
> > +		sa->sec_session = rte_security_session_create(ctx,
> > +				&sess_conf, pool);
> > +		if (sa->sec_session == NULL) {
> > +			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
> > +			return -1;
> > +		}
> >
> > -				if (sec_cap->action == sa->type &&
> > -				    sec_cap->protocol ==
> > -					RTE_SECURITY_PROTOCOL_IPSEC &&
> > -				    sec_cap->ipsec.mode ==
> > -
> > 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
> > -				    sec_cap->ipsec.direction == sa->direction)
> > -					break;
> > -				sec_cap++;
> > -			}
> > +		sec_cap = rte_security_capabilities_get(ctx);
> > +
> > +		/* iterate until ESP tunnel*/
> > +		while (sec_cap->action !=
> RTE_SECURITY_ACTION_TYPE_NONE)
> > {
> > +			if (sec_cap->action == sa->type &&
> > +				sec_cap->protocol ==
> > +				RTE_SECURITY_PROTOCOL_IPSEC &&
> > +				sec_cap->ipsec.mode ==
> > +				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL
> &&
> > +				sec_cap->ipsec.direction == sa->direction)
> > +				break;
> > +			sec_cap++;
> > +		}
> >
> > -			if (sec_cap->action ==
> > RTE_SECURITY_ACTION_TYPE_NONE) {
> > -				RTE_LOG(ERR, IPSEC,
> > +		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE)
> {
> > +			RTE_LOG(ERR, IPSEC,
> >  				"No suitable security capability found\n");
> >  				return -1;
> > -			}
> > +		}
> >
> > -			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > -			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > -
> > -			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > -			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > -			if (sa->flags & IP6_TUNNEL) {
> > -				sa->pattern[1].spec = &sa->ipv6_spec;
> > -				memcpy(sa->ipv6_spec.hdr.dst_addr,
> > -					sa->dst.ip.ip6.ip6_b, 16);
> > -				memcpy(sa->ipv6_spec.hdr.src_addr,
> > -				       sa->src.ip.ip6.ip6_b, 16);
> > -			} else {
> > -				sa->pattern[1].spec = &sa->ipv4_spec;
> > -				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > -				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > -			}
> > +		sa->ol_flags = sec_cap->ol_flags;
> > +		sa->security_ctx = ctx;
> > +		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> > +
> > +		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> > +		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
> > +		if (sa->flags & IP6_TUNNEL) {
> > +			sa->pattern[1].spec = &sa->ipv6_spec;
> > +			memcpy(sa->ipv6_spec.hdr.dst_addr,
> > +				sa->dst.ip.ip6.ip6_b, 16);
> > +			memcpy(sa->ipv6_spec.hdr.src_addr,
> > +			       sa->src.ip.ip6.ip6_b, 16);
> > +		} else {
> > +			sa->pattern[1].spec = &sa->ipv4_spec;
> > +			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
> > +			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
> > +		}
> >
> > -			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > -			sa->pattern[2].spec = &sa->esp_spec;
> > -			sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > -			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> > +		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
> > +		sa->pattern[2].spec = &sa->esp_spec;
> > +		sa->pattern[2].mask = &rte_flow_item_esp_mask;
> > +		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
> >
> > -			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> > +		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
> >
> > -			sa->action[0].type =
> > RTE_FLOW_ACTION_TYPE_SECURITY;
> > -			sa->action[0].conf = sa->sec_session;
> > +		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
> > +		sa->action[0].conf = sa->sec_session;
> >
> > -			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> > +		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
> >
> > -			sa->attr.egress = (sa->direction ==
> > +		sa->attr.egress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
> > -			sa->attr.ingress = (sa->direction ==
> > +		sa->attr.ingress = (sa->direction ==
> >
> > 	RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
> > -			if (sa->attr.ingress) {
> > -				uint8_t rss_key[40];
> > -				struct rte_eth_rss_conf rss_conf = {
> > -					.rss_key = rss_key,
> > -					.rss_key_len = 40,
> > -				};
> > -				struct rte_eth_dev *eth_dev;
> > -				uint16_t
> > queue[RTE_MAX_QUEUES_PER_PORT];
> > -				struct rte_flow_action_rss action_rss;
> > -				unsigned int i;
> > -				unsigned int j;
> > -
> > -				sa->action[2].type =
> > RTE_FLOW_ACTION_TYPE_END;
> > -				/* Try RSS. */
> > -				sa->action[1].type =
> > RTE_FLOW_ACTION_TYPE_RSS;
> > -				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > -				rte_eth_dev_rss_hash_conf_get(sa->portid,
> > -							      &rss_conf);
> > -				for (i = 0, j = 0;
> > -				     i < eth_dev->data->nb_rx_queues; ++i)
> > -					if (eth_dev->data->rx_queues[i])
> > -						queue[j++] = i;
> > +		if (sa->attr.ingress) {
> > +			uint8_t rss_key[40];
> > +			struct rte_eth_rss_conf rss_conf = {
> > +				.rss_key = rss_key,
> > +				.rss_key_len = 40,
> > +			};
> > +			struct rte_eth_dev *eth_dev;
> > +			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
> > +			struct rte_flow_action_rss action_rss;
> > +			unsigned int i;
> > +			unsigned int j;
> > +
> > +			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
> > +			/* Try RSS. */
> > +			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
> > +			sa->action[1].conf = &action_rss;
> > +			eth_dev = ctx->device;
> > +			rte_eth_dev_rss_hash_conf_get(sa->portid,
> > +						&rss_conf);
> > +			for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > +				++i)
> > +				if (eth_dev->data->rx_queues[i])
> > +					queue[j++] = i;
> Compilation error
> 
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> function 'create_sec_session':
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> error: this 'for' clause does not guard... [-Werror=misleading-indentation]
>     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
>     ^~~
> /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> note: ...this statement, but the latter is misleadingly indented as if it is
> guarded by the 'for'
>      action_rss = (struct rte_flow_action_rss){
>      ^~~~~~~~~~
> 
<snip>

I will send a v4.
What compiler are you using?

Regards,

Bernard.

Re: [dpdk-dev] [EXT] RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> > > +                   rte_eth_dev_rss_hash_conf_get(sa->portid,
> > > +                                           &rss_conf);
> > > +                   for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > > +                           ++i)
> > > +                           if (eth_dev->data->rx_queues[i])
> > > +                                   queue[j++] = i;
> > Compilation error
> >
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> > function 'create_sec_session':
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> > error: this 'for' clause does not guard... [-Werror=misleading-indentation]
> >     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> >     ^~~
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> > note: ...this statement, but the latter is misleadingly indented as if it is
> > guarded by the 'for'
> >      action_rss = (struct rte_flow_action_rss){
> >      ^~~~~~~~~~
> >
> <snip>
> 
> I will send a v4.
> What compiler are you using?

Cross compile for arm
gcc-linaro-6.1.1-2016.08-x86_64_aarch64-linux-gnu
> 
> Regards,
> 
> Bernard.
> 

Re: [dpdk-dev] [EXT] RE: [PATCH v3 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Akhil Goyal]

> > > +                   rte_eth_dev_rss_hash_conf_get(sa->portid,
> > > +                                           &rss_conf);
> > > +                   for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> > > +                           ++i)
> > > +                           if (eth_dev->data->rx_queues[i])
> > > +                                   queue[j++] = i;
> > Compilation error
> >
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c: In
> > function 'create_sec_session':
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:169:4:
> > error: this 'for' clause does not guard... [-Werror=misleading-indentation]
> >     for (i = 0, j = 0; i < eth_dev->data->nb_rx_queues;
> >     ^~~
> > /home/akhil/up/dpdk-next-crypto/examples/ipsec-secgw/ipsec.c:173:5:
> > note: ...this statement, but the latter is misleadingly indented as if it is
> > guarded by the 'for'
> >      action_rss = (struct rte_flow_action_rss){
> >      ^~~~~~~~~~
> >
> <snip>
> 
> I will send a v4.
> What compiler are you using?

Cross compile for arm
gcc-linaro-6.1.1-2016.08-x86_64_aarch64-linux-gnu
> 
> Regards,
> 
> Bernard.
> 

[dpdk-dev] [PATCH v4 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v4:
--------------
Fix indentation at line 173 in ipsec.c to solve compile error on ARM.

Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2:
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 581 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 501 insertions(+), 461 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v4 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v4:
--------------
Fix indentation at line 173 in ipsec.c to solve compile error on ARM.

Changes in v3:
--------------
The previous refactoring of the create_session() function has been dropped.
The create_session() function is now called from sa_init() at startup.

The following functions have been added:
crypto_devid_fill() in ipsec-secgw.c
check_cryptodev_capability() in ipsec.c
check_cryptodev_aead_capability() in ipsec.c
create_sec_session() and create_crypto_session() in ipsec.c

The create_session() function has been refactored to call
the create_sec_session() and create_crypto_session() functions.


Changes in v2:
--------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st packet dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 271 +++++++------
 examples/ipsec-secgw/ipsec.c                 | 581 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |  10 +-
 examples/ipsec-secgw/ipsec_process.c         |  38 +-
 examples/ipsec-secgw/sa.c                    |  42 +-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 501 insertions(+), 461 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 ++++++++--------
 examples/ipsec-secgw/ipsec.c         | 581 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 501 insertions(+), 441 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..83080f7 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
-					.types = rss_conf.rss_hf,
-					.key_len = rss_conf.rss_key_len,
-					.queue_num = j,
-					.key = rss_key,
-					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+			action_rss = (struct rte_flow_action_rss){
+				.types = rss_conf.rss_hf,
+				.key_len = rss_conf.rss_key_len,
+				.queue_num = j,
+				.key = rss_key,
+				.queue = queue,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v4 1/2] examples/ipsec-secgw: fix 1st packet dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule.

Refactor ipsec-secgw.c, sa.c, ipsec.h and ipsec.c to create
sessions at startup to fix the issue of the first packet
being dropped for inline crypto.

The create_session() function is now called at initialisation in
sa_add_rules() which is called from sa_init().
The return code for add_rules is checked.
Calls to create_session() in other functions are dropped.

Add crypto_devid_fill() in ipsec-secgw.c
Add max_session_size() in ipsec-secgw.c
Add check_cryptodev_capability() in ipsec.c
Add check_cryptodev_aead_capability() in ipsec.c
Add create_sec_session() and create_crypto_session() in ipsec.c

The crypto_dev_fill() function has been added to find the
enabled crypto devices.

The max_session_size() function has been added to calculate memory
requirements.

The check_cryptodev_capability() and check_cryptodev_aead_capability()
functions have been added to check that the SA is supported by the
crypto device.

The create_session() function is refactored to use the
create_sec_session() and create_crypto_session() functions.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call crypto_devid_fill()
and max_session_size() and to call session_pool_init() and
session_priv_pool_init().
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 271 ++++++++--------
 examples/ipsec-secgw/ipsec.c         | 581 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |  10 +-
 examples/ipsec-secgw/ipsec_process.c |  38 +--
 examples/ipsec-secgw/sa.c            |  42 ++-
 5 files changed, 501 insertions(+), 441 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index ffbd00b..cc8bb57 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -182,6 +182,14 @@ struct lcore_params {
 	uint8_t lcore_id;
 } __rte_cache_aligned;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 
 static struct lcore_params *lcore_params;
@@ -1623,13 +1631,27 @@ check_cryptodev_mask(uint8_t cdev_id)
 	return -1;
 }
 
+static void
+crypto_devid_fill(void)
+{
+	uint32_t i, n;
+
+	n = rte_cryptodev_count();
+
+	for (i = 0; i != n; i++) {
+		if (check_cryptodev_mask(i) == 0)
+			crypto_devid[crypto_dev_num++] = i;
+	}
+}
+
 static int32_t
 cryptodevs_init(void)
 {
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
+	uint32_t dev_max_sess;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1652,45 +1674,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1722,51 +1705,12 @@ cryptodevs_init(void)
 		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
 		dev_conf.nb_queue_pairs = qp;
 
-		uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
+		dev_max_sess = cdev_info.sym.max_nb_sessions;
 		if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
 			rte_exit(EXIT_FAILURE,
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1787,38 +1731,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_pool = sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1896,98 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2068,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2100,10 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	crypto_devid_fill();
+
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2113,17 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
+
+		if (!numa_on)
+			break;
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2141,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2163,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 4352cb8..83080f7 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -39,42 +39,17 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 	ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
 }
 
-int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+static int
+create_sec_session(struct ipsec_sa *sa, struct rte_mempool *pool)
 {
-	struct rte_cryptodev_info cdev_info;
-	unsigned long cdev_id_qp = 0;
+	const struct rte_security_capability *sec_cap;
+	struct rte_security_ctx *ctx;
 	int32_t ret = 0;
-	struct cdev_key key = { 0 };
-
-	key.lcore_id = (uint8_t)rte_lcore_id();
-
-	key.cipher_algo = (uint8_t)sa->cipher_algo;
-	key.auth_algo = (uint8_t)sa->auth_algo;
-	key.aead_algo = (uint8_t)sa->aead_algo;
-
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
-				"No cryptodev: core %u, cipher_algo %u, "
-				"auth_algo %u, aead_algo %u\n",
-				key.lcore_id,
-				key.cipher_algo,
-				key.auth_algo,
-				key.aead_algo);
-			return -1;
-		}
-	}
 
-	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
-			"%u qp %u\n", sa->spi,
-			ipsec_ctx->tbl[cdev_id_qp].id,
-			ipsec_ctx->tbl[cdev_id_qp].qp);
+	if ((sa == NULL) || (pool == NULL))
+		return -EINVAL;
 
-	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
-		struct rte_security_session_conf sess_conf = {
+	struct rte_security_session_conf sess_conf = {
 			.action_type = sa->type,
 			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 			{.ipsec = {
@@ -90,247 +65,340 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			} },
 			.crypto_xform = sa->xforms,
 			.userdata = NULL,
-
 		};
 
-		if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_cryptodev_get_sec_ctx(
-							ipsec_ctx->tbl[cdev_id_qp].id);
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n",
+				ret);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		sa->sec_session = rte_security_session_create(ctx,
+				&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC, "SEC Session init failed\n");
+			return -1;
+		}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		sec_cap = rte_security_capabilities_get(ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
 				return -1;
-			}
+		}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 
-			sa->attr.egress = (sa->direction ==
+		sa->attr.egress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
+		sa->attr.ingress = (sa->direction ==
 					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
-					.types = rss_conf.rss_hf,
-					.key_len = rss_conf.rss_key_len,
-					.queue_num = j,
-					.key = rss_key,
-					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid,
+						&rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+			action_rss = (struct rte_flow_action_rss){
+				.types = rss_conf.rss_hf,
+				.key_len = rss_conf.rss_key_len,
+				.queue_num = j,
+				.key = rss_key,
+				.queue = queue,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
 							sa->pattern, sa->action,
 							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
-				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
-					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
-					RTE_FLOW_ACTION_TYPE_END;
-			}
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+				(sa->ol_flags &
+				RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
+			sa->action[1].type =
+				RTE_FLOW_ACTION_TYPE_PASSTHRU;
+			sa->action[2].type =
+				RTE_FLOW_ACTION_TYPE_END;
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
-				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		sa->flow = rte_flow_create(sa->portid,
+			&sa->attr, sa->pattern, sa->action, &err);
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		struct rte_security_ctx *ctx =
+				(struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
+		const struct rte_security_capability *sec_cap;
 
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
-
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
-
-			sess_conf.userdata = (void *) sa;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
+		if (ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
+
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(ctx,
+			&sess_conf, pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+		sec_cap = rte_security_capabilities_get(ctx);
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"No capabilities registered\n");
-				return -1;
-			}
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
+			"No capabilities registered\n");
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
 					RTE_SECURITY_ACTION_TYPE_NONE) {
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+			if (sec_cap->action == sa->type &&
+				sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+				sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+				sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+			"No suitable security capability found\n");
+			return -1;
+		}
+
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = ctx;
+	} else
+		return -EINVAL;
+	return 0;
+}
+
+#define CDEV_IV_SIZE 12
+
+static int
+check_cryptodev_aead_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	cap_idx.algo.aead = ss->aead_algo;
+
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap == NULL)
+		return -ENOENT;
+
+	return rte_cryptodev_sym_capability_check_aead(cap,
+					ss->cipher_key_len,
+					ss->digest_len,
+					ss->aad_len,
+					CDEV_IV_SIZE);
+}
+
+static int
+check_cryptodev_capablity(const struct ipsec_sa *ss, uint8_t dev_id)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	const struct rte_cryptodev_symmetric_capability *cap;
+	uint16_t auth_iv_len;
+	int rc = -1;
+
+	if (ss == NULL)
+		return rc;
+
+	if (ss->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
+		return check_cryptodev_aead_capablity(ss, dev_id);
+
+	auth_iv_len = 0;
+
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	cap_idx.algo.auth = ss->auth_algo;
+	cap = rte_cryptodev_sym_capability_get(dev_id, &cap_idx);
+	if (cap != NULL) {
+		rc = rte_cryptodev_sym_capability_check_auth(
+				cap, ss->auth_key_len, ss->digest_len,
+				auth_iv_len);
+		if (rc == 0) {
+			cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+			cap_idx.algo.cipher = ss->cipher_algo;
+			cap = rte_cryptodev_sym_capability_get(dev_id,
+					&cap_idx);
+			if (cap != NULL)
+				rc = rte_cryptodev_sym_capability_check_cipher(
+						cap, ss->cipher_key_len,
+						ss->iv_len);
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
-	sa->cdev_id_qp = cdev_id_qp;
 
-	return 0;
+	return rc;
+}
+
+static int
+create_crypto_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	int32_t rc;
+	uint32_t devnum, i;
+	struct rte_cryptodev_sym_session *s;
+	uint8_t devid[RTE_CRYPTO_MAX_DEVS];
+
+	/* check which cryptodevs support SA */
+	devnum = 0;
+	for (i = 0; i < crypto_dev_num; i++) {
+		rc = check_cryptodev_capablity(sa, crypto_devid[i]);
+		if (rc == 0)
+			devid[devnum++] = crypto_devid[i];
+	}
+
+	if (devnum == 0)
+		return -ENODEV;
+
+	s = rte_cryptodev_sym_session_create(pool);
+	if (s == NULL)
+		return -ENOMEM;
+
+	/* initialize SA crypto session for all supported devices */
+	for (i = 0; i != devnum; i++) {
+		rc = rte_cryptodev_sym_session_init(devid[i], s, sa->xforms,
+			pool);
+		if (rc != 0)
+			break;
+	}
+
+	if (i == devnum) {
+		sa->crypto_session = s;
+		return 0;
+	}
+
+	/* failure, do cleanup */
+	while (i-- != 0)
+		rte_cryptodev_sym_session_clear(devid[i], s);
+
+	rte_cryptodev_sym_session_free(s);
+	return rc;
+}
+
+int
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool)
+{
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE)
+		return create_sec_session(sa, pool);
+	else
+		return create_crypto_session(sa, pool);
 }
 
 /*
@@ -393,13 +461,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			sym_cop = get_sym_cop(&priv->cop);
 			sym_cop->m_src = pkts[i];
 
@@ -412,13 +473,6 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_crypto_op_attach_sym_session(&priv->cop,
 					sa->crypto_session);
 
@@ -429,12 +483,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +491,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 99f49d6..804330c 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -83,6 +83,14 @@ struct app_sa_prm {
 
 extern struct app_sa_prm app_sa_prm;
 
+/*
+ * Number of enabled crypto devices
+ * This number is needed when checking crypto device capabilities
+ */
+extern uint8_t crypto_dev_num;
+/* array of crypto device ID's */
+extern uint8_t crypto_devid[RTE_CRYPTO_MAX_DEVS];
+
 struct ipsec_sa {
 	struct rte_ipsec_session ips; /* one session per sa for now */
 	uint32_t spi;
@@ -306,6 +314,6 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_session(struct ipsec_sa *sa, struct rte_mempool *pool);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..0df6969 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -86,39 +86,6 @@ enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
 	cqp->len = len;
 }
 
-static inline int
-fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
-	struct ipsec_sa *sa)
-{
-	int32_t rc;
-
-	/* setup crypto section */
-	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->crypto.ses = sa->crypto_session;
-	/* setup session action type */
-	} else {
-		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
-			if (rc != 0)
-				return rc;
-		}
-		ss->security.ses = sa->sec_session;
-		ss->security.ctx = sa->security_ctx;
-		ss->security.ol_flags = sa->ol_flags;
-	}
-
-	rc = rte_ipsec_session_prepare(ss);
-	if (rc != 0)
-		memset(ss, 0, sizeof(*ss));
-
-	return rc;
-}
-
 /*
  * group input packets byt the SA they belong to.
  */
@@ -219,9 +186,8 @@ ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
 
 		ips = &sa->ips;
 
-		/* no valid HW session for that SA, try to create one */
-		if (sa == NULL || (ips->crypto.ses == NULL &&
-				fill_ipsec_session(ips, ctx, sa) != 0))
+		/* no valid HW session for that SA */
+		if (sa == NULL || ips->crypto.ses == NULL)
 			k = 0;
 
 		/* process packets inline */
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index a7298a3..0f36f5b 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -774,14 +774,14 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 	return 0;
 }
 
-
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+	uint32_t nb_entries, uint32_t inbound, struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -902,6 +902,12 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			print_one_sa_rule(sa, inbound);
 		}
+		rc = create_session(sa, skt_ctx->session_pool);
+		if (rc != 0) {
+			RTE_LOG(ERR, IPSEC_ESP,
+					"create_session() failed\n");
+			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -909,16 +915,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1018,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1036,12 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	rc = rte_ipsec_session_prepare(ss);
+	if (rc != 0)
+		memset(ss, 0, sizeof(*ss));
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1076,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1141,7 +1155,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		rc = sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add inbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1161,7 +1178,10 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		rc = sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
+		if (rc != 0)
+			rte_exit(EXIT_FAILURE,
+				"failed to add outbound rules\n");
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v4 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v4 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100644
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100644
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

[dpdk-dev] [PATCH v5 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto. 
 
Changes in v5:
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.
 
Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++------------------
 examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++++--
 5 files changed, 245 insertions(+), 181 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 6c626fa..24876ba 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1628,7 +1628,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1651,45 +1651,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1727,45 +1688,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1786,39 +1708,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 7b85330..40a6123 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		} else if (
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
+			) {
+				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
+				return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
+
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
+
+	sa->cdev_id_qp = cdev_id_qp;
+
+	return 0;
+}
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	unsigned long cdev_id_qp = 0;
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
+		struct rte_security_session_conf sess_conf = {
+			.action_type = sa->type,
+			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+			{.ipsec = {
+				.spi = sa->spi,
+				.salt = sa->salt,
+				.options = { 0 },
+				.direction = sa->direction,
+				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+				.mode = (sa->flags == IP4_TUNNEL ||
+						sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+			} },
+			.crypto_xform = sa->xforms,
+			.userdata = NULL,
+		};
+
+		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
 			const struct rte_security_capability *sec_cap;
 			int ret = 0;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
+			sec_ctx = (struct rte_security_ctx *)
+							rte_eth_dev_get_sec_ctx(
+							sa->portid);
+			if (sec_ctx == NULL) {
+				RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+				return -1;
+			}
+
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			/* iterate until ESP tunnel*/
 			while (sec_cap->action !=
@@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 
 			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
@@ -196,7 +254,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				/* Try RSS. */
 				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
+				eth_dev = sec_ctx->device;
 				rte_eth_dev_rss_hash_conf_get(sa->portid,
 							      &rss_conf);
 				for (i = 0, j = 0;
@@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 		} else if (sa->type ==
 				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
 			const struct rte_security_capability *sec_cap;
 
-			if (ctx == NULL) {
+			sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(sa->portid);
+
+			if (sec_ctx == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"Ethernet device doesn't have security features registered\n");
 				return -1;
@@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 
 			sess_conf.userdata = (void *) sa;
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
+			sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
 			if (sa->sec_session == NULL) {
 				RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
 
-			sec_cap = rte_security_capabilities_get(ctx);
+			sec_cap = rte_security_capabilities_get(sec_ctx);
 
 			if (sec_cap == NULL) {
 				RTE_LOG(ERR, IPSEC,
@@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 			}
 
 			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			sa->security_ctx = sec_ctx;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
-
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
 	}
 	sa->cdev_id_qp = cdev_id_qp;
 
@@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index e9272d7..41bac0b 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -312,6 +312,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 8d47d1d..e8e55bf 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v5 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

Hi Bernard,
Few small comments below.
Konstantin

> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++------------------
>  examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
>  examples/ipsec-secgw/ipsec.h         |   5 +-
>  examples/ipsec-secgw/ipsec_process.c |   9 +-
>  examples/ipsec-secgw/sa.c            |  46 +++++--
>  5 files changed, 245 insertions(+), 181 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
> index 6c626fa..24876ba 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -1628,7 +1628,7 @@ cryptodevs_init(void)
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
>  	uint16_t idx, max_nb_qps, qp, i;
> -	int16_t cdev_id, port_id;
> +	int16_t cdev_id;
>  	struct rte_hash_parameters params = { 0 };
> 
>  	params.entries = CDEV_MAP_ENTRIES;
> @@ -1651,45 +1651,6 @@ cryptodevs_init(void)
> 
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
> -	uint32_t max_sess_sz = 0, sess_sz;
> -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> -		void *sec_ctx;
> -
> -		/* Get crypto priv session size */
> -		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -
> -		/*
> -		 * If crypto device is security capable, need to check the
> -		 * size of security session as well.
> -		 */
> -
> -		/* Get security context of the crypto device */
> -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		/* Get size of security session */
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		void *sec_ctx;
> -
> -		if ((enabled_port_mask & (1 << port_id)) == 0)
> -			continue;
> -
> -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> -		if (sec_ctx == NULL)
> -			continue;
> -
> -		sess_sz = rte_security_session_get_size(sec_ctx);
> -		if (sess_sz > max_sess_sz)
> -			max_sess_sz = sess_sz;
> -	}
> -
>  	idx = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> @@ -1727,45 +1688,6 @@ cryptodevs_init(void)
>  				"Device does not support at least %u "
>  				"sessions", CDEV_MP_NB_OBJS);
> 
> -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_%u", dev_conf.socket_id);
> -			sess_mp = rte_cryptodev_sym_session_pool_create(
> -					mp_name, CDEV_MP_NB_OBJS,
> -					0, CDEV_MP_CACHE_SZ, 0,
> -					dev_conf.socket_id);
> -			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> -			struct rte_mempool *sess_mp;
> -
> -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -					"sess_mp_priv_%u", dev_conf.socket_id);
> -			sess_mp = rte_mempool_create(mp_name,
> -					CDEV_MP_NB_OBJS,
> -					max_sess_sz,
> -					CDEV_MP_CACHE_SZ,
> -					0, NULL, NULL, NULL,
> -					NULL, dev_conf.socket_id,
> -					0);
> -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> -					sess_mp;
> -		}
> -
> -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> -				!socket_ctx[dev_conf.socket_id].session_pool)
> -			rte_exit(EXIT_FAILURE,
> -				"Cannot create session pool on socket %d\n",
> -				dev_conf.socket_id);
> -		else
> -			printf("Allocated session pool on socket %d\n",
> -					dev_conf.socket_id);
> -
>  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
>  			rte_panic("Failed to initialize cryptodev %u\n",
>  					cdev_id);
> @@ -1786,39 +1708,6 @@ cryptodevs_init(void)
>  					cdev_id);
>  	}
> 
> -	/* create session pools for eth devices that implement security */
> -	RTE_ETH_FOREACH_DEV(port_id) {
> -		if ((enabled_port_mask & (1 << port_id)) &&
> -				rte_eth_dev_get_sec_ctx(port_id)) {
> -			int socket_id = rte_eth_dev_socket_id(port_id);
> -
> -			if (!socket_ctx[socket_id].session_priv_pool) {
> -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> -				struct rte_mempool *sess_mp;
> -
> -				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> -						"sess_mp_%u", socket_id);
> -				sess_mp = rte_mempool_create(mp_name,
> -						(CDEV_MP_NB_OBJS * 2),
> -						max_sess_sz,
> -						CDEV_MP_CACHE_SZ,
> -						0, NULL, NULL, NULL,
> -						NULL, socket_id,
> -						0);
> -				if (sess_mp == NULL)
> -					rte_exit(EXIT_FAILURE,
> -						"Cannot create session pool "
> -						"on socket %d\n", socket_id);
> -				else
> -					printf("Allocated session pool "
> -						"on socket %d\n", socket_id);
> -				socket_ctx[socket_id].session_priv_pool =
> -						sess_mp;
> -			}
> -		}
> -	}
> -
> -
>  	printf("\n");
> 
>  	return 0;
> @@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
>  	printf("\n");
>  }
> 
> +static size_t
> +max_session_size(void)
> +{
> +	size_t max_sz, sz;
> +	void *sec_ctx;
> +	int16_t cdev_id, port_id, n;
> +
> +	max_sz = 0;
> +	n =  rte_cryptodev_count();
> +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +		/*
> +		 * If crypto device is security capable, need to check the
> +		 * size of security session as well.
> +		 */
> +
> +		/* Get security context of the crypto device */
> +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		/* Get size of security session */
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	RTE_ETH_FOREACH_DEV(port_id) {
> +		if ((enabled_port_mask & (1 << port_id)) == 0)
> +			continue;
> +
> +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> +		if (sec_ctx == NULL)
> +			continue;
> +
> +		sz = rte_security_session_get_size(sec_ctx);
> +		if (sz > max_sz)
> +			max_sz = sz;
> +	}
> +
> +	return max_sz;
> +}
> +
> +static void
> +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_%u", socket_id);
> +	sess_mp = rte_cryptodev_sym_session_pool_create(
> +			mp_name, CDEV_MP_NB_OBJS,
> +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> +			socket_id);
> +	ctx->session_pool = sess_mp;
> +
> +	if (ctx->session_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session pool on socket %d\n", socket_id);
> +	else
> +		printf("Allocated session pool on socket %d\n",	socket_id);
> +}
> +
> +static void
> +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> +	size_t sess_sz)
> +{
> +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> +	struct rte_mempool *sess_mp;
> +
> +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> +			"sess_mp_priv_%u", socket_id);
> +	sess_mp = rte_mempool_create(mp_name,
> +			CDEV_MP_NB_OBJS,
> +			sess_sz,
> +			CDEV_MP_CACHE_SZ,
> +			0, NULL, NULL, NULL,
> +			NULL, socket_id,
> +			0);
> +	ctx->session_priv_pool = sess_mp;
> +
> +	if (ctx->session_priv_pool == NULL)
> +		rte_exit(EXIT_FAILURE,
> +			"Cannot init session priv pool on socket %d\n",
> +			socket_id);
> +	else
> +		printf("Allocated session priv pool on socket %d\n",
> +			socket_id);
> +}
> +
>  static void
>  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
>  {
> @@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
>  	uint32_t lcore_id;
> +	uint32_t i;
>  	uint8_t socket_id;
>  	uint16_t portid;
>  	uint64_t req_rx_offloads, req_tx_offloads;
> +	size_t sess_sz;
> 
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
> @@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
> 
>  	nb_lcores = rte_lcore_count();
> 
> -	/* Replicate each context per socket */
> +	sess_sz = max_session_size();
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
>  		else
>  			socket_id = 0;
> 
> +		/* mbuf_pool is initialised by the pool_init() function*/
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		/* initilaze SPD */
> -		sp4_init(&socket_ctx[socket_id], socket_id);
> -
> -		sp6_init(&socket_ctx[socket_id], socket_id);
> -
> -		/* initilaze SAD */
> -		sa_init(&socket_ctx[socket_id], socket_id);
> -
> -		rt_init(&socket_ctx[socket_id], socket_id);
> -
>  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> +			sess_sz);
>  	}
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
> @@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
>  		if ((enabled_port_mask & (1 << portid)) == 0)
>  			continue;
> 
> -		/* Start device */
> +		/*
> +		 * Start device
> +		 * note: device must be started before a flow rule
> +		 * can be installed.
> +		 */
>  		ret = rte_eth_dev_start(portid);
>  		if (ret < 0)
>  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> @@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
>  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
>  	}
> 
> +	/* Replicate each context per socket */
> +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> +		socket_id = rte_socket_id_by_idx(i);
> +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> +			(socket_ctx[socket_id].sa_in == NULL) &&
> +			(socket_ctx[socket_id].sa_out == NULL)) {
> +			sa_init(&socket_ctx[socket_id], socket_id);
> +			sp4_init(&socket_ctx[socket_id], socket_id);
> +			sp6_init(&socket_ctx[socket_id], socket_id);
> +			rt_init(&socket_ctx[socket_id], socket_id);
> +		}
> +	}
> +
>  	check_all_ports_link_status(enabled_port_mask);
> 
>  	/* launch per-lcore init on every lcore */
> diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
> index 7b85330..40a6123 100644
> --- a/examples/ipsec-secgw/ipsec.c
> +++ b/examples/ipsec-secgw/ipsec.c
> @@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
>  }
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  {
>  	struct rte_cryptodev_info cdev_info;
>  	unsigned long cdev_id_qp = 0;


Actually looking at it a bit closer:

if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {

I think we need to remove that line above, otherwise cdev_id_qp
Would not be initialized for LOOKASIDE_PROTO.
BTW, as I can see, same problem exists in current code.


                ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
                                (void **)&cdev_id_qp);
                if (ret < 0) {
                        RTE_LOG(ERR, IPSEC,
                                "No cryptodev: core %u, cipher_algo %u, "
                                "auth_algo %u, aead_algo %u\n",
                                key.lcore_id,
                                key.cipher_algo,
                                key.auth_algo,
                                key.aead_algo);
                        return -1;
                }
        }

> @@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> -		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +		} else if (
> +			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
> +			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> +			) {
> +				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
> +				return -1;
> +		}
> +	} else {
> +		sa->crypto_session = rte_cryptodev_sym_session_create(
> +				ipsec_ctx->session_pool);
> +		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> +				sa->crypto_session, sa->xforms,
> +				ipsec_ctx->session_priv_pool);
> +
> +		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> +				&cdev_info);
> +	}
> +
> +	sa->cdev_id_qp = cdev_id_qp;
> +
> +	return 0;
> +}
> +
> +int
> +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
> +{
> +	unsigned long cdev_id_qp = 0;

This var is not really used in this function.

> +	int32_t ret = 0;
> +	struct rte_security_ctx *sec_ctx;
> +
> +	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
> +		sa->spi, sa->portid);
> +
> +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {

I think this always be true here and could be removed.


> +		struct rte_security_session_conf sess_conf = {
> +			.action_type = sa->type,
> +			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> +			{.ipsec = {
> +				.spi = sa->spi,
> +				.salt = sa->salt,
> +				.options = { 0 },
> +				.direction = sa->direction,
> +				.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
> +				.mode = (sa->flags == IP4_TUNNEL ||
> +						sa->flags == IP6_TUNNEL) ?
> +					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
> +					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
> +			} },
> +			.crypto_xform = sa->xforms,
> +			.userdata = NULL,
> +		};
> +
> +		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
>  			struct rte_flow_error err;
> -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> -							rte_eth_dev_get_sec_ctx(
> -							sa->portid);
>  			const struct rte_security_capability *sec_cap;
>  			int ret = 0;
> 
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_priv_pool);
> +			sec_ctx = (struct rte_security_ctx *)
> +							rte_eth_dev_get_sec_ctx(
> +							sa->portid);
> +			if (sec_ctx == NULL) {
> +				RTE_LOG(ERR, IPSEC,
> +				" rte_eth_dev_get_sec_ctx failed\n");
> +				return -1;
> +			}
> +
> +			sa->sec_session = rte_security_session_create(sec_ctx,
> +					&sess_conf, skt_ctx->session_pool);
>  			if (sa->sec_session == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +			sec_cap = rte_security_capabilities_get(sec_ctx);
> 
>  			/* iterate until ESP tunnel*/
>  			while (sec_cap->action !=
> @@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
> 
>  			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +			sa->security_ctx = sec_ctx;
>  			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> 
>  			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> @@ -196,7 +254,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  				/* Try RSS. */
>  				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
>  				sa->action[1].conf = &action_rss;
> -				eth_dev = ctx->device;
> +				eth_dev = sec_ctx->device;
>  				rte_eth_dev_rss_hash_conf_get(sa->portid,
>  							      &rss_conf);
>  				for (i = 0, j = 0;
> @@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
>  		} else if (sa->type ==
>  				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> -			struct rte_security_ctx *ctx =
> -					(struct rte_security_ctx *)
> -					rte_eth_dev_get_sec_ctx(sa->portid);
>  			const struct rte_security_capability *sec_cap;
> 
> -			if (ctx == NULL) {
> +			sec_ctx = (struct rte_security_ctx *)
> +					rte_eth_dev_get_sec_ctx(sa->portid);
> +
> +			if (sec_ctx == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"Ethernet device doesn't have security features registered\n");
>  				return -1;
> @@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> 
>  			sess_conf.userdata = (void *) sa;
> 
> -			sa->sec_session = rte_security_session_create(ctx,
> -					&sess_conf, ipsec_ctx->session_pool);
> +			sa->sec_session = rte_security_session_create(sec_ctx,
> +					&sess_conf, skt_ctx->session_pool);
>  			if (sa->sec_session == NULL) {
>  				RTE_LOG(ERR, IPSEC,
>  				"SEC Session init failed: err: %d\n", ret);
>  				return -1;
>  			}
> 
> -			sec_cap = rte_security_capabilities_get(ctx);
> +			sec_cap = rte_security_capabilities_get(sec_ctx);
> 
>  			if (sec_cap == NULL) {
>  				RTE_LOG(ERR, IPSEC,
> @@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
>  			}
> 
>  			sa->ol_flags = sec_cap->ol_flags;
> -			sa->security_ctx = ctx;
> +			sa->security_ctx = sec_ctx;
>  		}
> -	} else {
> -		sa->crypto_session = rte_cryptodev_sym_session_create(
> -				ipsec_ctx->session_pool);
> -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> -				sa->crypto_session, sa->xforms,
> -				ipsec_ctx->session_priv_pool);
> -
> -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> -				&cdev_info);
>  	}
>  	sa->cdev_id_qp = cdev_id_qp;
> 
> @@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			rte_prefetch0(&priv->sym_cop);
> 
>  			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> +				create_lookaside_session(ipsec_ctx, sa)) {
>  				rte_pktmbuf_free(pkts[i]);
>  				continue;
>  			}
> @@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			rte_prefetch0(&priv->sym_cop);
> 
>  			if ((unlikely(sa->crypto_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> +				create_lookaside_session(ipsec_ctx, sa)) {
>  				rte_pktmbuf_free(pkts[i]);
>  				continue;
>  			}
> @@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  			}
>  			break;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
>  			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
>  				rte_security_set_pkt_metadata(
> @@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
>  						sa->sec_session, pkts[i], NULL);
>  			continue;
>  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> +			RTE_ASSERT(sa->sec_session != NULL);
>  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
>  			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> 
>  			rte_prefetch0(&priv->sym_cop);
> -
> -			if ((unlikely(sa->sec_session == NULL)) &&
> -					create_session(ipsec_ctx, sa)) {
> -				rte_pktmbuf_free(pkts[i]);
> -				continue;
> -			}
> -
>  			rte_security_attach_session(&priv->cop,
>  					sa->sec_session);
> 
> diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
> index e9272d7..41bac0b 100644
> --- a/examples/ipsec-secgw/ipsec.h
> +++ b/examples/ipsec-secgw/ipsec.h
> @@ -312,6 +312,9 @@ void
>  enqueue_cop_burst(struct cdev_qp *cqp);
> 
>  int
> -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> +
> +int
> +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
> 
>  #endif /* __IPSEC_H__ */
> diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
> index 3f9cacb..868f1a2 100644
> --- a/examples/ipsec-secgw/ipsec_process.c
> +++ b/examples/ipsec-secgw/ipsec_process.c
> @@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
>  	/* setup crypto section */
>  	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
>  		if (sa->crypto_session == NULL) {
> -			rc = create_session(ctx, sa);
> +			rc = create_lookaside_session(ctx, sa);
>  			if (rc != 0)
>  				return rc;
>  		}
>  		ss->crypto.ses = sa->crypto_session;
>  	/* setup session action type */
> -	} else {
> +	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
>  		if (sa->sec_session == NULL) {
> -			rc = create_session(ctx, sa);
> +			rc = create_lookaside_session(ctx, sa);
>  			if (rc != 0)
>  				return rc;
>  		}
>  		ss->security.ses = sa->sec_session;
>  		ss->security.ctx = sa->security_ctx;
>  		ss->security.ol_flags = sa->ol_flags;
> -	}
> +	} else
> +		RTE_ASSERT(0);
> 
>  	rc = rte_ipsec_session_prepare(ss);
>  	if (rc != 0)
> diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> index 8d47d1d..e8e55bf 100644
> --- a/examples/ipsec-secgw/sa.c
> +++ b/examples/ipsec-secgw/sa.c
> @@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
> 
>  static int
>  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries, uint32_t inbound)
> +		uint32_t nb_entries, uint32_t inbound,
> +		struct socket_ctx *skt_ctx)
>  {
>  	struct ipsec_sa *sa;
>  	uint32_t i, idx;
>  	uint16_t iv_length, aad_length;
> +	int32_t rc;
> 
>  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
>  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
> @@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  			sa->xforms = &sa_ctx->xf[idx].a;
> 
> +			if (sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> +				sa->type ==
> +				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> +				rc = create_inline_session(skt_ctx, sa);
> +				if (rc != 0) {
> +					RTE_LOG(ERR, IPSEC_ESP,
> +						"create_inline_session() failed\n");
> +					return -EINVAL;
> +				}
> +			}
>  			print_one_sa_rule(sa, inbound);
>  		} else {
>  			switch (sa->cipher_algo) {
> @@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> 
>  static inline int
>  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
>  }
> 
>  static inline int
>  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> -		uint32_t nb_entries)
> +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
>  {
> -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
>  }
> 
>  /*
> @@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
>  	return 0;
>  }
> 
> -static void
> +static int
>  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  	const struct ipsec_sa *lsa)
>  {
> +	int32_t rc = 0;
> +
>  	ss->sa = sa;
>  	ss->type = lsa->type;
> 
> @@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
>  		ss->security.ctx = lsa->security_ctx;
>  		ss->security.ol_flags = lsa->ol_flags;
>  	}
> +
> +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> +		if (ss->security.ses != NULL) {
> +			rc = rte_ipsec_session_prepare(ss);
> +			if (rc != 0)
> +				memset(ss, 0, sizeof(*ss));
> +		}
> +	}
> +
> +	return rc;
>  }
> 
>  /*
> @@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
>  	if (rc < 0)
>  		return rc;
> 
> -	fill_ipsec_session(&lsa->ips, sa, lsa);
> -	return 0;
> +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> +	return rc;
>  }
> 
>  /*
> @@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
> @@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
>  				"context %s in socket %d\n", rte_errno,
>  				name, socket_id);
> 
> -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> 
>  		if (app_sa_prm.enable != 0) {
>  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> --
> 2.7.4

Re: [dpdk-dev] [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Konstantin,

> -----Original Message-----
> From: Ananyev, Konstantin
> Sent: Tuesday, June 11, 2019 4:29 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; dev@dpdk.org;
> akhil.goyal@nxp.com
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v5 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline
> crypto
> 
> Hi Bernard,
> Few small comments below.
> Konstantin
> 
> > Inline crypto installs a flow rule in the NIC. This flow rule must be
> > installed before the first inbound packet is received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session() and
> > create_lookaside_session(). The create_inline_session() function uses
> > the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call max_session_size() and
> > to call session_pool_init() and session_priv_pool_init() earlier.
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are now
> > called after the ports have been started.
> >
> > The rte_ipsec_session_prepare() function is called in
> > fill_ipsec_session() for inline which is called from the ipsec_sa_init() function.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> >  examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++++++++++--------------
> ----
> >  examples/ipsec-secgw/ipsec.c         | 122 ++++++++++++------
> >  examples/ipsec-secgw/ipsec.h         |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c |   9 +-
> >  examples/ipsec-secgw/sa.c            |  46 +++++--
> >  5 files changed, 245 insertions(+), 181 deletions(-)
> >
> > diff --git a/examples/ipsec-secgw/ipsec-secgw.c
> > b/examples/ipsec-secgw/ipsec-secgw.c
> > index 6c626fa..24876ba 100644
> > --- a/examples/ipsec-secgw/ipsec-secgw.c
> > +++ b/examples/ipsec-secgw/ipsec-secgw.c
> > @@ -1628,7 +1628,7 @@ cryptodevs_init(void)
> >  	struct rte_cryptodev_config dev_conf;
> >  	struct rte_cryptodev_qp_conf qp_conf;
> >  	uint16_t idx, max_nb_qps, qp, i;
> > -	int16_t cdev_id, port_id;
> > +	int16_t cdev_id;
> >  	struct rte_hash_parameters params = { 0 };
> >
> >  	params.entries = CDEV_MAP_ENTRIES;
> > @@ -1651,45 +1651,6 @@ cryptodevs_init(void)
> >
> >  	printf("lcore/cryptodev/qp mappings:\n");
> >
> > -	uint32_t max_sess_sz = 0, sess_sz;
> > -	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> > -		void *sec_ctx;
> > -
> > -		/* Get crypto priv session size */
> > -		sess_sz =
> rte_cryptodev_sym_get_private_session_size(cdev_id);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -
> > -		/*
> > -		 * If crypto device is security capable, need to check the
> > -		 * size of security session as well.
> > -		 */
> > -
> > -		/* Get security context of the crypto device */
> > -		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		/* Get size of security session */
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		void *sec_ctx;
> > -
> > -		if ((enabled_port_mask & (1 << port_id)) == 0)
> > -			continue;
> > -
> > -		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > -		if (sec_ctx == NULL)
> > -			continue;
> > -
> > -		sess_sz = rte_security_session_get_size(sec_ctx);
> > -		if (sess_sz > max_sess_sz)
> > -			max_sess_sz = sess_sz;
> > -	}
> > -
> >  	idx = 0;
> >  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
> >  		struct rte_cryptodev_info cdev_info; @@ -1727,45 +1688,6
> @@
> > cryptodevs_init(void)
> >  				"Device does not support at least %u "
> >  				"sessions", CDEV_MP_NB_OBJS);
> >
> > -		if (!socket_ctx[dev_conf.socket_id].session_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_%u", dev_conf.socket_id);
> > -			sess_mp = rte_cryptodev_sym_session_pool_create(
> > -					mp_name, CDEV_MP_NB_OBJS,
> > -					0, CDEV_MP_CACHE_SZ, 0,
> > -					dev_conf.socket_id);
> > -			socket_ctx[dev_conf.socket_id].session_pool =
> sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
> > -			char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -			struct rte_mempool *sess_mp;
> > -
> > -			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > -					"sess_mp_priv_%u",
> dev_conf.socket_id);
> > -			sess_mp = rte_mempool_create(mp_name,
> > -					CDEV_MP_NB_OBJS,
> > -					max_sess_sz,
> > -					CDEV_MP_CACHE_SZ,
> > -					0, NULL, NULL, NULL,
> > -					NULL, dev_conf.socket_id,
> > -					0);
> > -			socket_ctx[dev_conf.socket_id].session_priv_pool =
> > -					sess_mp;
> > -		}
> > -
> > -		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
> > -				!socket_ctx[dev_conf.socket_id].session_pool)
> > -			rte_exit(EXIT_FAILURE,
> > -				"Cannot create session pool on socket %d\n",
> > -				dev_conf.socket_id);
> > -		else
> > -			printf("Allocated session pool on socket %d\n",
> > -					dev_conf.socket_id);
> > -
> >  		if (rte_cryptodev_configure(cdev_id, &dev_conf))
> >  			rte_panic("Failed to initialize cryptodev %u\n",
> >  					cdev_id);
> > @@ -1786,39 +1708,6 @@ cryptodevs_init(void)
> >  					cdev_id);
> >  	}
> >
> > -	/* create session pools for eth devices that implement security */
> > -	RTE_ETH_FOREACH_DEV(port_id) {
> > -		if ((enabled_port_mask & (1 << port_id)) &&
> > -				rte_eth_dev_get_sec_ctx(port_id)) {
> > -			int socket_id = rte_eth_dev_socket_id(port_id);
> > -
> > -			if (!socket_ctx[socket_id].session_priv_pool) {
> > -				char mp_name[RTE_MEMPOOL_NAMESIZE];
> > -				struct rte_mempool *sess_mp;
> > -
> > -				snprintf(mp_name,
> RTE_MEMPOOL_NAMESIZE,
> > -						"sess_mp_%u", socket_id);
> > -				sess_mp = rte_mempool_create(mp_name,
> > -						(CDEV_MP_NB_OBJS * 2),
> > -						max_sess_sz,
> > -						CDEV_MP_CACHE_SZ,
> > -						0, NULL, NULL, NULL,
> > -						NULL, socket_id,
> > -						0);
> > -				if (sess_mp == NULL)
> > -					rte_exit(EXIT_FAILURE,
> > -						"Cannot create session pool "
> > -						"on socket %d\n", socket_id);
> > -				else
> > -					printf("Allocated session pool "
> > -						"on socket %d\n", socket_id);
> > -				socket_ctx[socket_id].session_priv_pool =
> > -						sess_mp;
> > -			}
> > -		}
> > -	}
> > -
> > -
> >  	printf("\n");
> >
> >  	return 0;
> > @@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t
> req_rx_offloads, uint64_t req_tx_offloads)
> >  	printf("\n");
> >  }
> >
> > +static size_t
> > +max_session_size(void)
> > +{
> > +	size_t max_sz, sz;
> > +	void *sec_ctx;
> > +	int16_t cdev_id, port_id, n;
> > +
> > +	max_sz = 0;
> > +	n =  rte_cryptodev_count();
> > +	for (cdev_id = 0; cdev_id != n; cdev_id++) {
> > +		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +		/*
> > +		 * If crypto device is security capable, need to check the
> > +		 * size of security session as well.
> > +		 */
> > +
> > +		/* Get security context of the crypto device */
> > +		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		/* Get size of security session */
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	RTE_ETH_FOREACH_DEV(port_id) {
> > +		if ((enabled_port_mask & (1 << port_id)) == 0)
> > +			continue;
> > +
> > +		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
> > +		if (sec_ctx == NULL)
> > +			continue;
> > +
> > +		sz = rte_security_session_get_size(sec_ctx);
> > +		if (sz > max_sz)
> > +			max_sz = sz;
> > +	}
> > +
> > +	return max_sz;
> > +}
> > +
> > +static void
> > +session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t
> > +sess_sz) {
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_%u", socket_id);
> > +	sess_mp = rte_cryptodev_sym_session_pool_create(
> > +			mp_name, CDEV_MP_NB_OBJS,
> > +			sess_sz, CDEV_MP_CACHE_SZ, 0,
> > +			socket_id);
> > +	ctx->session_pool = sess_mp;
> > +
> > +	if (ctx->session_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session pool on socket %d\n", socket_id);
> > +	else
> > +		printf("Allocated session pool on socket %d\n",	socket_id);
> > +}
> > +
> > +static void
> > +session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
> > +	size_t sess_sz)
> > +{
> > +	char mp_name[RTE_MEMPOOL_NAMESIZE];
> > +	struct rte_mempool *sess_mp;
> > +
> > +	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
> > +			"sess_mp_priv_%u", socket_id);
> > +	sess_mp = rte_mempool_create(mp_name,
> > +			CDEV_MP_NB_OBJS,
> > +			sess_sz,
> > +			CDEV_MP_CACHE_SZ,
> > +			0, NULL, NULL, NULL,
> > +			NULL, socket_id,
> > +			0);
> > +	ctx->session_priv_pool = sess_mp;
> > +
> > +	if (ctx->session_priv_pool == NULL)
> > +		rte_exit(EXIT_FAILURE,
> > +			"Cannot init session priv pool on socket %d\n",
> > +			socket_id);
> > +	else
> > +		printf("Allocated session priv pool on socket %d\n",
> > +			socket_id);
> > +}
> > +
> >  static void
> >  pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t
> > nb_mbuf)  { @@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)  {
> >  	int32_t ret;
> >  	uint32_t lcore_id;
> > +	uint32_t i;
> >  	uint8_t socket_id;
> >  	uint16_t portid;
> >  	uint64_t req_rx_offloads, req_tx_offloads;
> > +	size_t sess_sz;
> >
> >  	/* init EAL */
> >  	ret = rte_eal_init(argc, argv);
> > @@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
> >
> >  	nb_lcores = rte_lcore_count();
> >
> > -	/* Replicate each context per socket */
> > +	sess_sz = max_session_size();
> > +
> >  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> >  		if (rte_lcore_is_enabled(lcore_id) == 0)
> >  			continue;
> > @@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
> >  		else
> >  			socket_id = 0;
> >
> > +		/* mbuf_pool is initialised by the pool_init() function*/
> >  		if (socket_ctx[socket_id].mbuf_pool)
> >  			continue;
> >
> > -		/* initilaze SPD */
> > -		sp4_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		sp6_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		/* initilaze SAD */
> > -		sa_init(&socket_ctx[socket_id], socket_id);
> > -
> > -		rt_init(&socket_ctx[socket_id], socket_id);
> > -
> >  		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> > +		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
> > +		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
> > +			sess_sz);
> >  	}
> >
> >  	RTE_ETH_FOREACH_DEV(portid) {
> > @@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
> >  		if ((enabled_port_mask & (1 << portid)) == 0)
> >  			continue;
> >
> > -		/* Start device */
> > +		/*
> > +		 * Start device
> > +		 * note: device must be started before a flow rule
> > +		 * can be installed.
> > +		 */
> >  		ret = rte_eth_dev_start(portid);
> >  		if (ret < 0)
> >  			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
> > @@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
> >  			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback,
> NULL);
> >  	}
> >
> > +	/* Replicate each context per socket */
> > +	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
> > +		socket_id = rte_socket_id_by_idx(i);
> > +		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
> > +			(socket_ctx[socket_id].sa_in == NULL) &&
> > +			(socket_ctx[socket_id].sa_out == NULL)) {
> > +			sa_init(&socket_ctx[socket_id], socket_id);
> > +			sp4_init(&socket_ctx[socket_id], socket_id);
> > +			sp6_init(&socket_ctx[socket_id], socket_id);
> > +			rt_init(&socket_ctx[socket_id], socket_id);
> > +		}
> > +	}
> > +
> >  	check_all_ports_link_status(enabled_port_mask);
> >
> >  	/* launch per-lcore init on every lcore */ diff --git
> > a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index
> > 7b85330..40a6123 100644
> > --- a/examples/ipsec-secgw/ipsec.c
> > +++ b/examples/ipsec-secgw/ipsec.c
> > @@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct
> > rte_security_ipsec_xform *ipsec)  }
> >
> >  int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> > +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa
> > +*sa)
> >  {
> >  	struct rte_cryptodev_info cdev_info;
> >  	unsigned long cdev_id_qp = 0;
> 
> 
> Actually looking at it a bit closer:
> 
> if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> 
> I think we need to remove that line above, otherwise cdev_id_qp Would not be
> initialized for LOOKASIDE_PROTO.
> BTW, as I can see, same problem exists in current code.

Will do.

>                 ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
>                                 (void **)&cdev_id_qp);
>                 if (ret < 0) {
>                         RTE_LOG(ERR, IPSEC,
>                                 "No cryptodev: core %u, cipher_algo %u, "
>                                 "auth_algo %u, aead_algo %u\n",
>                                 key.lcore_id,
>                                 key.cipher_algo,
>                                 key.auth_algo,
>                                 key.aead_algo);
>                         return -1;
>                 }
>         }
> 
> > @@ -108,23 +108,81 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> > -		} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > +		} else if (
> > +			(sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
> > +			(sa->type ==
> RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> > +			) {
> > +				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
> > +				return -1;
> > +		}
> > +	} else {
> > +		sa->crypto_session = rte_cryptodev_sym_session_create(
> > +				ipsec_ctx->session_pool);
> > +		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> > +				sa->crypto_session, sa->xforms,
> > +				ipsec_ctx->session_priv_pool);
> > +
> > +		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> > +				&cdev_info);
> > +	}
> > +
> > +	sa->cdev_id_qp = cdev_id_qp;
> > +
> > +	return 0;
> > +}
> > +
> > +int
> > +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa
> > +*sa) {
> > +	unsigned long cdev_id_qp = 0;
> 
> This var is not really used in this function.

Will remove cdev_id_qp.
 
> > +	int32_t ret = 0;
> > +	struct rte_security_ctx *sec_ctx;
> > +
> > +	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port
> %u\n",
> > +		sa->spi, sa->portid);
> > +
> > +	if (sa->type != RTE_SECURITY_ACTION_TYPE_NONE) {
> 
> I think this always be true here and could be removed.

Will do.
 
> > +		struct rte_security_session_conf sess_conf = {
> > +			.action_type = sa->type,
> > +			.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
> > +			{.ipsec = {
> > +				.spi = sa->spi,
> > +				.salt = sa->salt,
> > +				.options = { 0 },
> > +				.direction = sa->direction,
> > +				.proto =
> RTE_SECURITY_IPSEC_SA_PROTO_ESP,
> > +				.mode = (sa->flags == IP4_TUNNEL ||
> > +						sa->flags == IP6_TUNNEL) ?
> > +
> 	RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
> > +
> 	RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
> > +			} },
> > +			.crypto_xform = sa->xforms,
> > +			.userdata = NULL,
> > +		};
> > +
> > +		if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
> {
> >  			struct rte_flow_error err;
> > -			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
> > -
> 	rte_eth_dev_get_sec_ctx(
> > -							sa->portid);
> >  			const struct rte_security_capability *sec_cap;
> >  			int ret = 0;
> >
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx-
> >session_priv_pool);
> > +			sec_ctx = (struct rte_security_ctx *)
> > +
> 	rte_eth_dev_get_sec_ctx(
> > +							sa->portid);
> > +			if (sec_ctx == NULL) {
> > +				RTE_LOG(ERR, IPSEC,
> > +				" rte_eth_dev_get_sec_ctx failed\n");
> > +				return -1;
> > +			}
> > +
> > +			sa->sec_session = rte_security_session_create(sec_ctx,
> > +					&sess_conf, skt_ctx->session_pool);
> >  			if (sa->sec_session == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +			sec_cap = rte_security_capabilities_get(sec_ctx);
> >
> >  			/* iterate until ESP tunnel*/
> >  			while (sec_cap->action !=
> > @@ -147,7 +205,7 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >
> >  			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > +			sa->security_ctx = sec_ctx;
> >  			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> >
> >  			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
> @@ -196,7 +254,7 @@
> > create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
> >  				/* Try RSS. */
> >  				sa->action[1].type =
> RTE_FLOW_ACTION_TYPE_RSS;
> >  				sa->action[1].conf = &action_rss;
> > -				eth_dev = ctx->device;
> > +				eth_dev = sec_ctx->device;
> >  				rte_eth_dev_rss_hash_conf_get(sa->portid,
> >  							      &rss_conf);
> >  				for (i = 0, j = 0;
> > @@ -252,12 +310,12 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >  		} else if (sa->type ==
> >
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
> > -			struct rte_security_ctx *ctx =
> > -					(struct rte_security_ctx *)
> > -					rte_eth_dev_get_sec_ctx(sa->portid);
> >  			const struct rte_security_capability *sec_cap;
> >
> > -			if (ctx == NULL) {
> > +			sec_ctx = (struct rte_security_ctx *)
> > +					rte_eth_dev_get_sec_ctx(sa->portid);
> > +
> > +			if (sec_ctx == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"Ethernet device doesn't have security features
> registered\n");
> >  				return -1;
> > @@ -279,15 +337,15 @@ create_session(struct ipsec_ctx *ipsec_ctx,
> > struct ipsec_sa *sa)
> >
> >  			sess_conf.userdata = (void *) sa;
> >
> > -			sa->sec_session = rte_security_session_create(ctx,
> > -					&sess_conf, ipsec_ctx->session_pool);
> > +			sa->sec_session = rte_security_session_create(sec_ctx,
> > +					&sess_conf, skt_ctx->session_pool);
> >  			if (sa->sec_session == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> >  				"SEC Session init failed: err: %d\n", ret);
> >  				return -1;
> >  			}
> >
> > -			sec_cap = rte_security_capabilities_get(ctx);
> > +			sec_cap = rte_security_capabilities_get(sec_ctx);
> >
> >  			if (sec_cap == NULL) {
> >  				RTE_LOG(ERR, IPSEC,
> > @@ -316,17 +374,8 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct
> ipsec_sa *sa)
> >  			}
> >
> >  			sa->ol_flags = sec_cap->ol_flags;
> > -			sa->security_ctx = ctx;
> > +			sa->security_ctx = sec_ctx;
> >  		}
> > -	} else {
> > -		sa->crypto_session = rte_cryptodev_sym_session_create(
> > -				ipsec_ctx->session_pool);
> > -		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
> > -				sa->crypto_session, sa->xforms,
> > -				ipsec_ctx->session_priv_pool);
> > -
> > -		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
> > -				&cdev_info);
> >  	}
> >  	sa->cdev_id_qp = cdev_id_qp;
> >
> > @@ -395,7 +444,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			rte_prefetch0(&priv->sym_cop);
> >
> >  			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > +				create_lookaside_session(ipsec_ctx, sa)) {
> >  				rte_pktmbuf_free(pkts[i]);
> >  				continue;
> >  			}
> > @@ -414,7 +463,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			rte_prefetch0(&priv->sym_cop);
> >
> >  			if ((unlikely(sa->crypto_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > +				create_lookaside_session(ipsec_ctx, sa)) {
> >  				rte_pktmbuf_free(pkts[i]);
> >  				continue;
> >  			}
> > @@ -429,12 +478,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  			}
> >  			break;
> >  		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
> > -			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > -				rte_pktmbuf_free(pkts[i]);
> > -				continue;
> > -			}
> > -
> > +			RTE_ASSERT(sa->sec_session != NULL);
> >  			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
> >  			if (sa->ol_flags &
> RTE_SECURITY_TX_OLOAD_NEED_MDATA)
> >  				rte_security_set_pkt_metadata(
> > @@ -442,17 +486,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct
> ipsec_ctx *ipsec_ctx,
> >  						sa->sec_session, pkts[i],
> NULL);
> >  			continue;
> >  		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
> > +			RTE_ASSERT(sa->sec_session != NULL);
> >  			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
> >  			priv->cop.status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> >
> >  			rte_prefetch0(&priv->sym_cop);
> > -
> > -			if ((unlikely(sa->sec_session == NULL)) &&
> > -					create_session(ipsec_ctx, sa)) {
> > -				rte_pktmbuf_free(pkts[i]);
> > -				continue;
> > -			}
> > -
> >  			rte_security_attach_session(&priv->cop,
> >  					sa->sec_session);
> >
> > diff --git a/examples/ipsec-secgw/ipsec.h
> > b/examples/ipsec-secgw/ipsec.h index e9272d7..41bac0b 100644
> > --- a/examples/ipsec-secgw/ipsec.h
> > +++ b/examples/ipsec-secgw/ipsec.h
> > @@ -312,6 +312,9 @@ void
> >  enqueue_cop_burst(struct cdev_qp *cqp);
> >
> >  int
> > -create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
> > +create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa
> > +*sa);
> > +
> > +int
> > +create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa
> > +*sa);
> >
> >  #endif /* __IPSEC_H__ */
> > diff --git a/examples/ipsec-secgw/ipsec_process.c
> > b/examples/ipsec-secgw/ipsec_process.c
> > index 3f9cacb..868f1a2 100644
> > --- a/examples/ipsec-secgw/ipsec_process.c
> > +++ b/examples/ipsec-secgw/ipsec_process.c
> > @@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct
> ipsec_ctx *ctx,
> >  	/* setup crypto section */
> >  	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
> >  		if (sa->crypto_session == NULL) {
> > -			rc = create_session(ctx, sa);
> > +			rc = create_lookaside_session(ctx, sa);
> >  			if (rc != 0)
> >  				return rc;
> >  		}
> >  		ss->crypto.ses = sa->crypto_session;
> >  	/* setup session action type */
> > -	} else {
> > +	} else if (sa->type ==
> RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
> > +{
> >  		if (sa->sec_session == NULL) {
> > -			rc = create_session(ctx, sa);
> > +			rc = create_lookaside_session(ctx, sa);
> >  			if (rc != 0)
> >  				return rc;
> >  		}
> >  		ss->security.ses = sa->sec_session;
> >  		ss->security.ctx = sa->security_ctx;
> >  		ss->security.ol_flags = sa->ol_flags;
> > -	}
> > +	} else
> > +		RTE_ASSERT(0);
> >
> >  	rc = rte_ipsec_session_prepare(ss);
> >  	if (rc != 0)
> > diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
> > index 8d47d1d..e8e55bf 100644
> > --- a/examples/ipsec-secgw/sa.c
> > +++ b/examples/ipsec-secgw/sa.c
> > @@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t
> > inbound)
> >
> >  static int
> >  sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries, uint32_t inbound)
> > +		uint32_t nb_entries, uint32_t inbound,
> > +		struct socket_ctx *skt_ctx)
> >  {
> >  	struct ipsec_sa *sa;
> >  	uint32_t i, idx;
> >  	uint16_t iv_length, aad_length;
> > +	int32_t rc;
> >
> >  	/* for ESN upper 32 bits of SQN also need to be part of AAD */
> >  	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0; @@
> > -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> > ipsec_sa entries[],
> >
> >  			sa->xforms = &sa_ctx->xf[idx].a;
> >
> > +			if (sa->type ==
> > +
> 	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
> > +				sa->type ==
> > +
> 	RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
> > +				rc = create_inline_session(skt_ctx, sa);
> > +				if (rc != 0) {
> > +					RTE_LOG(ERR, IPSEC_ESP,
> > +						"create_inline_session()
> failed\n");
> > +					return -EINVAL;
> > +				}
> > +			}
> >  			print_one_sa_rule(sa, inbound);
> >  		} else {
> >  			switch (sa->cipher_algo) {
> > @@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct
> > ipsec_sa entries[],
> >
> >  static inline int
> >  sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries)
> > +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
> >  {
> > -	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
> > +	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
> >  }
> >
> >  static inline int
> >  sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
> > -		uint32_t nb_entries)
> > +		uint32_t nb_entries, struct socket_ctx *skt_ctx)
> >  {
> > -	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
> > +	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
> >  }
> >
> >  /*
> > @@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm,
> const struct ipsec_sa *ss,
> >  	return 0;
> >  }
> >
> > -static void
> > +static int
> >  fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
> >  	const struct ipsec_sa *lsa)
> >  {
> > +	int32_t rc = 0;
> > +
> >  	ss->sa = sa;
> >  	ss->type = lsa->type;
> >
> > @@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss,
> struct rte_ipsec_sa *sa,
> >  		ss->security.ctx = lsa->security_ctx;
> >  		ss->security.ol_flags = lsa->ol_flags;
> >  	}
> > +
> > +	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
> > +		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
> {
> > +		if (ss->security.ses != NULL) {
> > +			rc = rte_ipsec_session_prepare(ss);
> > +			if (rc != 0)
> > +				memset(ss, 0, sizeof(*ss));
> > +		}
> > +	}
> > +
> > +	return rc;
> >  }
> >
> >  /*
> > @@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct
> rte_ipsec_sa *sa, uint32_t sa_size)
> >  	if (rc < 0)
> >  		return rc;
> >
> > -	fill_ipsec_session(&lsa->ips, sa, lsa);
> > -	return 0;
> > +	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
> > +	return rc;
> >  }
> >
> >  /*
> > @@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
> >  				"context %s in socket %d\n", rte_errno,
> >  				name, socket_id);
> >
> > -		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
> > +		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
> >
> >  		if (app_sa_prm.enable != 0) {
> >  			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in, @@ -
> 1186,7
> > +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
> >  				"context %s in socket %d\n", rte_errno,
> >  				name, socket_id);
> >
> > -		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
> > +		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
> >
> >  		if (app_sa_prm.enable != 0) {
> >  			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
> > --
> > 2.7.4

Thanks for the review.

Regards,

Bernard.

[dpdk-dev] [PATCH v6 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v6:
--------------
Rebased to latest master.
Minor changes to the following functions in ipsec.c:
create_lookaside_session()
create_inline_session()

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
 examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 403 insertions(+), 370 deletions(-)

Changes in v5: 
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.
 
Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2: 
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 243 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 123 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 246 insertions(+), 200 deletions(-)


-- 
2.7.4

[dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 ++++++++++---------
 examples/ipsec-secgw/ipsec.c         | 449 +++++++++++++++++++----------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++-
 5 files changed, 403 insertions(+), 350 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index 6c626fa..24876ba 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1628,7 +1628,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	params.entries = CDEV_MAP_ENTRIES;
@@ -1651,45 +1651,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1727,45 +1688,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1786,39 +1708,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -1984,6 +1873,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2064,9 +2046,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2094,7 +2078,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2104,20 +2089,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2135,7 +2114,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2153,6 +2136,19 @@ main(int32_t argc, char **argv)
 			RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index 7b85330..c06ddd6 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -53,19 +53,17 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 	key.auth_algo = (uint8_t)sa->auth_algo;
 	key.aead_algo = (uint8_t)sa->aead_algo;
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
+	ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
+			(void **)&cdev_id_qp);
+	if (ret < 0) {
+		RTE_LOG(ERR, IPSEC,
 				"No cryptodev: core %u, cipher_algo %u, "
 				"auth_algo %u, aead_algo %u\n",
 				key.lcore_id,
 				key.cipher_algo,
 				key.auth_algo,
 				key.aead_algo);
-			return -1;
-		}
+		return -1;
 	}
 
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
@@ -108,227 +106,267 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
+		} else if (
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) ||
+			(sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
+			) {
+				RTE_LOG(ERR, IPSEC, "Inline not supported\n");
 				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+	sa->cdev_id_qp = cdev_id_qp;
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+	return 0;
+}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-			if (sa->flags & IP6_TUNNEL) {
-				sa->pattern[1].spec = &sa->ipv6_spec;
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else {
-				sa->pattern[1].spec = &sa->ipv4_spec;
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+	struct rte_security_session_conf sess_conf = {
+		.action_type = sa->type,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		{.ipsec = {
+			.spi = sa->spi,
+			.salt = sa->salt,
+			.options = { 0 },
+			.direction = sa->direction,
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = (sa->flags == IP4_TUNNEL ||
+					sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+		} },
+		.crypto_xform = sa->xforms,
+		.userdata = NULL,
+	};
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		const struct rte_security_capability *sec_cap;
+		int ret = 0;
+
+		sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(
+					sa->portid);
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+			return -1;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->sec_session = rte_security_session_create(sec_ctx,
+				&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+				"No suitable security capability found\n");
+			return -1;
+		}
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+		sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+		if (sa->flags & IP6_TUNNEL) {
+			sa->pattern[1].spec = &sa->ipv6_spec;
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else {
+			sa->pattern[1].spec = &sa->ipv4_spec;
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->attr.egress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
+
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+
+		sa->attr.egress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
+		sa->attr.ingress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = sec_ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid, &rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+
+			action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+			   (sa->ol_flags &
 				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
+			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
+			sa->action[2].type =
 					RTE_FLOW_ACTION_TYPE_END;
-			}
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
+		sa->flow = rte_flow_create(sa->portid,
 				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		const struct rte_security_capability *sec_cap;
 
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
+		sec_ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sess_conf.userdata = (void *) sa;
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+			return -1;
+		}
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"No capabilities registered\n");
-				return -1;
-			}
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
-
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
-				return -1;
-			}
-
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			return -1;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
 
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
 	}
-	sa->cdev_id_qp = cdev_id_qp;
+
+	sa->cdev_id_qp = 0;
 
 	return 0;
 }
@@ -395,7 +433,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -414,7 +452,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -429,12 +467,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -442,17 +475,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index e9272d7..41bac0b 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -312,6 +312,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 8d47d1d..e8e55bf 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -777,11 +777,13 @@ check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -834,6 +836,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -909,16 +922,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1012,10 +1025,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1028,6 +1043,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1062,8 +1088,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1166,7 +1192,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1186,7 +1212,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Wednesday, June 12, 2019 3:52 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>; stable@dpdk.org
> Subject: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> 
> Inline crypto installs a flow rule in the NIC. This flow
> rule must be installed before the first inbound packet is
> received.
> 
> The create_session() function installs the flow rule,
> create_session() has been refactored into create_inline_session()
> and create_lookaside_session(). The create_inline_session() function
> uses the socket_ctx data and is now called at initialisation in
> sa_add_rules().
> 
> The max_session_size() function has been added to calculate memory
> requirements.
> 
> The cryprodev_init() function has been refactored to drop calls to
> rte_mempool_create() and to drop calculation of memory requirements.
> 
> The main() function has been refactored to call max_session_size() and
> to call session_pool_init() and session_priv_pool_init() earlier.
> The ports are started now before adding a flow rule in main().
> The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> now called after the ports have been started.
> 
> The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> for inline which is called from the ipsec_sa_init() function.
> 
> Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> --
> 2.7.4

Re: [dpdk-dev] [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Ananyev, Konstantin]

> -----Original Message-----
> From: Iremonger, Bernard
> Sent: Wednesday, June 12, 2019 3:52 PM
> To: dev@dpdk.org; Ananyev, Konstantin <konstantin.ananyev@intel.com>; akhil.goyal@nxp.com
> Cc: Iremonger, Bernard <bernard.iremonger@intel.com>; stable@dpdk.org
> Subject: [PATCH v6 2/2] examples/ipsec-secgw/test: fix inline test scripts
> 
> Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
> to get around the bug where the first inbound packet is dropped
> for inline crypto.
> 
> Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> ---

Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

> --
> 2.7.4

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

Hi Bernard,

> >
> > Inline crypto installs a flow rule in the NIC. This flow
> > rule must be installed before the first inbound packet is
> > received.
> >
> > The create_session() function installs the flow rule,
> > create_session() has been refactored into create_inline_session()
> > and create_lookaside_session(). The create_inline_session() function
> > uses the socket_ctx data and is now called at initialisation in
> > sa_add_rules().
> >
> > The max_session_size() function has been added to calculate memory
> > requirements.
> >
> > The cryprodev_init() function has been refactored to drop calls to
> > rte_mempool_create() and to drop calculation of memory requirements.
> >
> > The main() function has been refactored to call max_session_size() and
> > to call session_pool_init() and session_priv_pool_init() earlier.
> > The ports are started now before adding a flow rule in main().
> > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > now called after the ports have been started.
> >
> > The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
> > for inline which is called from the ipsec_sa_init() function.
> >
> > Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
> > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > ---
> 
> Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> 

This patch need to be rebased.

However, I will try to work on your v5 in next few days if I get time. If I don't get time to fix that, will need you to send a rebased version of this patch.

Thanks,
Akhil

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, July 3, 2019 11:05 AM
> To: Ananyev, Konstantin <konstantin.ananyev@intel.com>; Iremonger,
> Bernard <bernard.iremonger@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for
> inline crypto
> 
> Hi Bernard,
> 
> > >
> > > Inline crypto installs a flow rule in the NIC. This flow rule must
> > > be installed before the first inbound packet is received.
> > >
> > > The create_session() function installs the flow rule,
> > > create_session() has been refactored into create_inline_session()
> > > and create_lookaside_session(). The create_inline_session() function
> > > uses the socket_ctx data and is now called at initialisation in
> > > sa_add_rules().
> > >
> > > The max_session_size() function has been added to calculate memory
> > > requirements.
> > >
> > > The cryprodev_init() function has been refactored to drop calls to
> > > rte_mempool_create() and to drop calculation of memory requirements.
> > >
> > > The main() function has been refactored to call max_session_size()
> > > and to call session_pool_init() and session_priv_pool_init() earlier.
> > > The ports are started now before adding a flow rule in main().
> > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > > now called after the ports have been started.
> > >
> > > The rte_ipsec_session_prepare() function is called in
> > > fill_ipsec_session() for inline which is called from the ipsec_sa_init()
> function.
> > >
> > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > offload")
> > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > application")
> > > Cc: stable@dpdk.org
> > >
> > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > ---
> >
> > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> >
> 
> This patch need to be rebased.
> 
> However, I will try to work on your v5 in next few days if I get time. If I don't
> get time to fix that, will need you to send a rebased version of this patch.
> 
> Thanks,
> Akhil
> 

v6 is the latest version
I will send a v7 rebased on 19.08.rc1

Regards,

Bernard

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

> 
> Hi Akhil,
> 
> >
> > Hi Bernard,
> >
> > > >
> > > > Inline crypto installs a flow rule in the NIC. This flow rule must
> > > > be installed before the first inbound packet is received.
> > > >
> > > > The create_session() function installs the flow rule,
> > > > create_session() has been refactored into create_inline_session()
> > > > and create_lookaside_session(). The create_inline_session() function
> > > > uses the socket_ctx data and is now called at initialisation in
> > > > sa_add_rules().
> > > >
> > > > The max_session_size() function has been added to calculate memory
> > > > requirements.
> > > >
> > > > The cryprodev_init() function has been refactored to drop calls to
> > > > rte_mempool_create() and to drop calculation of memory requirements.
> > > >
> > > > The main() function has been refactored to call max_session_size()
> > > > and to call session_pool_init() and session_priv_pool_init() earlier.
> > > > The ports are started now before adding a flow rule in main().
> > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
> > > > now called after the ports have been started.
> > > >
> > > > The rte_ipsec_session_prepare() function is called in
> > > > fill_ipsec_session() for inline which is called from the ipsec_sa_init()
> > function.
> > > >
> > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > offload")
> > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > application")
> > > > Cc: stable@dpdk.org
> > > >
> > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > ---
> > >
> > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > >
> >
> > This patch need to be rebased.
> >
> > However, I will try to work on your v5 in next few days if I get time. If I don't
> > get time to fix that, will need you to send a rebased version of this patch.
> >
> > Thanks,
> > Akhil
> >
> 
> v6 is the latest version
> I will send a v7 rebased on 19.08.rc1

I was talking about your previous which was for both lookaside and inline cases to create sessions at init time.
If I get time, I will fix that or will ask you to rebase this one.
> 
> Regards,
> 
> Bernard
> 

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Iremonger, Bernard]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal [mailto:akhil.goyal@nxp.com]
> Sent: Wednesday, July 3, 2019 11:18 AM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>; Ananyev,
> Konstantin <konstantin.ananyev@intel.com>; dev@dpdk.org
> Cc: stable@dpdk.org
> Subject: RE: [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for
> inline crypto
> 
> 
> 
> >
> > Hi Akhil,
> >
> > >
> > > Hi Bernard,
> > >
> > > > >
> > > > > Inline crypto installs a flow rule in the NIC. This flow rule
> > > > > must be installed before the first inbound packet is received.
> > > > >
> > > > > The create_session() function installs the flow rule,
> > > > > create_session() has been refactored into
> > > > > create_inline_session() and create_lookaside_session(). The
> > > > > create_inline_session() function uses the socket_ctx data and is
> > > > > now called at initialisation in sa_add_rules().
> > > > >
> > > > > The max_session_size() function has been added to calculate
> > > > > memory requirements.
> > > > >
> > > > > The cryprodev_init() function has been refactored to drop calls
> > > > > to
> > > > > rte_mempool_create() and to drop calculation of memory
> requirements.
> > > > >
> > > > > The main() function has been refactored to call
> > > > > max_session_size() and to call session_pool_init() and
> session_priv_pool_init() earlier.
> > > > > The ports are started now before adding a flow rule in main().
> > > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions
> > > > > are now called after the ports have been started.
> > > > >
> > > > > The rte_ipsec_session_prepare() function is called in
> > > > > fill_ipsec_session() for inline which is called from the
> > > > > ipsec_sa_init()
> > > function.
> > > > >
> > > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > > offload")
> > > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > > application")
> > > > > Cc: stable@dpdk.org
> > > > >
> > > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > > ---
> > > >
> > > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > >
> > >
> > > This patch need to be rebased.
> > >
> > > However, I will try to work on your v5 in next few days if I get
> > > time. If I don't get time to fix that, will need you to send a rebased
> version of this patch.
> > >
> > > Thanks,
> > > Akhil
> > >
> >
> > v6 is the latest version
> > I will send a v7 rebased on 19.08.rc1
> 
> I was talking about your previous which was for both lookaside and inline
> cases to create sessions at init time.
> If I get time, I will fix that or will ask you to rebase this one.
> >
> > Regards,
> >
> > Bernard
> >

They previous one was v4, where we could not test the lookaside code.
Should I rebase the v4 and send as v7?

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v6 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Akhil Goyal]

> 
> Hi Akhil,
> 
> >
> >
> >
> > >
> > > Hi Akhil,
> > >
> > > >
> > > > Hi Bernard,
> > > >
> > > > > >
> > > > > > Inline crypto installs a flow rule in the NIC. This flow rule
> > > > > > must be installed before the first inbound packet is received.
> > > > > >
> > > > > > The create_session() function installs the flow rule,
> > > > > > create_session() has been refactored into
> > > > > > create_inline_session() and create_lookaside_session(). The
> > > > > > create_inline_session() function uses the socket_ctx data and is
> > > > > > now called at initialisation in sa_add_rules().
> > > > > >
> > > > > > The max_session_size() function has been added to calculate
> > > > > > memory requirements.
> > > > > >
> > > > > > The cryprodev_init() function has been refactored to drop calls
> > > > > > to
> > > > > > rte_mempool_create() and to drop calculation of memory
> > requirements.
> > > > > >
> > > > > > The main() function has been refactored to call
> > > > > > max_session_size() and to call session_pool_init() and
> > session_priv_pool_init() earlier.
> > > > > > The ports are started now before adding a flow rule in main().
> > > > > > The sa_init(), sp4_init(), sp6_init() and rt_init() functions
> > > > > > are now called after the ports have been started.
> > > > > >
> > > > > > The rte_ipsec_session_prepare() function is called in
> > > > > > fill_ipsec_session() for inline which is called from the
> > > > > > ipsec_sa_init()
> > > > function.
> > > > > >
> > > > > > Fixes: ec17993a145a ("examples/ipsec-secgw: support security
> > > > > > offload")
> > > > > > Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample
> > > > > > application")
> > > > > > Cc: stable@dpdk.org
> > > > > >
> > > > > > Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
> > > > > > ---
> > > > >
> > > > > Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > > Tested-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
> > > > >
> > > >
> > > > This patch need to be rebased.
> > > >
> > > > However, I will try to work on your v5 in next few days if I get
> > > > time. If I don't get time to fix that, will need you to send a rebased
> > version of this patch.
> > > >
> > > > Thanks,
> > > > Akhil
> > > >
> > >
> > > v6 is the latest version
> > > I will send a v7 rebased on 19.08.rc1
> >
> > I was talking about your previous which was for both lookaside and inline
> > cases to create sessions at init time.
> > If I get time, I will fix that or will ask you to rebase this one.
> > >
> > > Regards,
> > >
> > > Bernard
> > >
> 
> They previous one was v4, where we could not test the lookaside code.
> Should I rebase the v4 and send as v7?
> 
I will let you know, if your v4 can work well or your v6 will be rebased and applied.


> Regards,
> 
> Bernard.
> 

[dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Bernard Iremonger]

This patchset fixes the issue of the first inbound packet
being dropped for inline crypto.

Changes in v7:
--------------
Rebased to DPDK 19.08-rc1

Changes in v6:
--------------
Rebased to latest master.
Minor changes to the following functions in ipsec.c:
create_lookaside_session()
create_inline_session()

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
 examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 403 insertions(+), 370 deletions(-)

Changes in v5:
-------------
The v2 patchset has been rebased to the latest master.
The v4 patchset has been dropped as it caused issues with the lookaside code
which we are unable to test.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
 examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 +++--
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
 7 files changed, 245 insertions(+), 201 deletions(-)

Changes in v2:
-------------
The first three patches of the v1 have been squashed.
The commit message for the squashed patch has been updated.
Patches 4,5 and 6 of the v1 have been dropped from this patchset.
A patch to fix the test scripts has been added.

Bernard Iremonger (2):
  examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
  examples/ipsec-secgw/test: fix inline test scripts

 examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
 examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
 examples/ipsec-secgw/ipsec.h                 |   5 +-
 examples/ipsec-secgw/ipsec_process.c         |   9 +-
 examples/ipsec-secgw/sa.c                    |  46 ++-
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
 7 files changed, 405 insertions(+), 375 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH v7 1/2] examples/ipsec-secgw: fix 1st pkt dropped for inline crypto

[Bernard Iremonger]

Inline crypto installs a flow rule in the NIC. This flow
rule must be installed before the first inbound packet is
received.

The create_session() function installs the flow rule,
create_session() has been refactored into create_inline_session()
and create_lookaside_session(). The create_inline_session() function
uses the socket_ctx data and is now called at initialisation in
sa_add_rules().

The max_session_size() function has been added to calculate memory
requirements.

The cryprodev_init() function has been refactored to drop calls to
rte_mempool_create() and to drop calculation of memory requirements.

The main() function has been refactored to call max_session_size() and
to call session_pool_init() and session_priv_pool_init() earlier.
The ports are started now before adding a flow rule in main().
The sa_init(), sp4_init(), sp6_init() and rt_init() functions are
now called after the ports have been started.

The rte_ipsec_session_prepare() function is called in fill_ipsec_session()
for inline which is called from the ipsec_sa_init() function.

Fixes: ec17993a145a ("examples/ipsec-secgw: support security offload")
Fixes: d299106e8e31 ("examples/ipsec-secgw: add IPsec sample application")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/ipsec-secgw.c   | 244 +++++++++----------
 examples/ipsec-secgw/ipsec.c         | 456 ++++++++++++++++++-----------------
 examples/ipsec-secgw/ipsec.h         |   5 +-
 examples/ipsec-secgw/ipsec_process.c |   9 +-
 examples/ipsec-secgw/sa.c            |  46 +++-
 5 files changed, 405 insertions(+), 355 deletions(-)

diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index b1ecbb9..f4819ce 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -1802,7 +1802,7 @@ cryptodevs_init(void)
 	struct rte_cryptodev_config dev_conf;
 	struct rte_cryptodev_qp_conf qp_conf;
 	uint16_t idx, max_nb_qps, qp, i;
-	int16_t cdev_id, port_id;
+	int16_t cdev_id;
 	struct rte_hash_parameters params = { 0 };
 
 	const uint64_t mseg_flag = multi_seg_required() ?
@@ -1828,45 +1828,6 @@ cryptodevs_init(void)
 
 	printf("lcore/cryptodev/qp mappings:\n");
 
-	uint32_t max_sess_sz = 0, sess_sz;
-	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
-		void *sec_ctx;
-
-		/* Get crypto priv session size */
-		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-
-		/*
-		 * If crypto device is security capable, need to check the
-		 * size of security session as well.
-		 */
-
-		/* Get security context of the crypto device */
-		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		/* Get size of security session */
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-	RTE_ETH_FOREACH_DEV(port_id) {
-		void *sec_ctx;
-
-		if ((enabled_port_mask & (1 << port_id)) == 0)
-			continue;
-
-		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
-		if (sec_ctx == NULL)
-			continue;
-
-		sess_sz = rte_security_session_get_size(sec_ctx);
-		if (sess_sz > max_sess_sz)
-			max_sess_sz = sess_sz;
-	}
-
 	idx = 0;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
 		struct rte_cryptodev_info cdev_info;
@@ -1911,45 +1872,6 @@ cryptodevs_init(void)
 				"Device does not support at least %u "
 				"sessions", CDEV_MP_NB_OBJS);
 
-		if (!socket_ctx[dev_conf.socket_id].session_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_%u", dev_conf.socket_id);
-			sess_mp = rte_cryptodev_sym_session_pool_create(
-					mp_name, CDEV_MP_NB_OBJS,
-					0, CDEV_MP_CACHE_SZ, 0,
-					dev_conf.socket_id);
-			socket_ctx[dev_conf.socket_id].session_pool = sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool) {
-			char mp_name[RTE_MEMPOOL_NAMESIZE];
-			struct rte_mempool *sess_mp;
-
-			snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-					"sess_mp_priv_%u", dev_conf.socket_id);
-			sess_mp = rte_mempool_create(mp_name,
-					CDEV_MP_NB_OBJS,
-					max_sess_sz,
-					CDEV_MP_CACHE_SZ,
-					0, NULL, NULL, NULL,
-					NULL, dev_conf.socket_id,
-					0);
-			socket_ctx[dev_conf.socket_id].session_priv_pool =
-					sess_mp;
-		}
-
-		if (!socket_ctx[dev_conf.socket_id].session_priv_pool ||
-				!socket_ctx[dev_conf.socket_id].session_pool)
-			rte_exit(EXIT_FAILURE,
-				"Cannot create session pool on socket %d\n",
-				dev_conf.socket_id);
-		else
-			printf("Allocated session pool on socket %d\n",
-					dev_conf.socket_id);
-
 		if (rte_cryptodev_configure(cdev_id, &dev_conf))
 			rte_panic("Failed to initialize cryptodev %u\n",
 					cdev_id);
@@ -1970,39 +1892,6 @@ cryptodevs_init(void)
 					cdev_id);
 	}
 
-	/* create session pools for eth devices that implement security */
-	RTE_ETH_FOREACH_DEV(port_id) {
-		if ((enabled_port_mask & (1 << port_id)) &&
-				rte_eth_dev_get_sec_ctx(port_id)) {
-			int socket_id = rte_eth_dev_socket_id(port_id);
-
-			if (!socket_ctx[socket_id].session_priv_pool) {
-				char mp_name[RTE_MEMPOOL_NAMESIZE];
-				struct rte_mempool *sess_mp;
-
-				snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
-						"sess_mp_%u", socket_id);
-				sess_mp = rte_mempool_create(mp_name,
-						(CDEV_MP_NB_OBJS * 2),
-						max_sess_sz,
-						CDEV_MP_CACHE_SZ,
-						0, NULL, NULL, NULL,
-						NULL, socket_id,
-						0);
-				if (sess_mp == NULL)
-					rte_exit(EXIT_FAILURE,
-						"Cannot create session pool "
-						"on socket %d\n", socket_id);
-				else
-					printf("Allocated session pool "
-						"on socket %d\n", socket_id);
-				socket_ctx[socket_id].session_priv_pool =
-						sess_mp;
-			}
-		}
-	}
-
-
 	printf("\n");
 
 	return 0;
@@ -2174,6 +2063,99 @@ port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
 	printf("\n");
 }
 
+static size_t
+max_session_size(void)
+{
+	size_t max_sz, sz;
+	void *sec_ctx;
+	int16_t cdev_id, port_id, n;
+
+	max_sz = 0;
+	n =  rte_cryptodev_count();
+	for (cdev_id = 0; cdev_id != n; cdev_id++) {
+		sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
+		if (sz > max_sz)
+			max_sz = sz;
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		if ((enabled_port_mask & (1 << port_id)) == 0)
+			continue;
+
+		sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		sz = rte_security_session_get_size(sec_ctx);
+		if (sz > max_sz)
+			max_sz = sz;
+	}
+
+	return max_sz;
+}
+
+static void
+session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_%u", socket_id);
+	sess_mp = rte_cryptodev_sym_session_pool_create(
+			mp_name, CDEV_MP_NB_OBJS,
+			sess_sz, CDEV_MP_CACHE_SZ, 0,
+			socket_id);
+	ctx->session_pool = sess_mp;
+
+	if (ctx->session_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session pool on socket %d\n", socket_id);
+	else
+		printf("Allocated session pool on socket %d\n",	socket_id);
+}
+
+static void
+session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
+	size_t sess_sz)
+{
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+	struct rte_mempool *sess_mp;
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+			"sess_mp_priv_%u", socket_id);
+	sess_mp = rte_mempool_create(mp_name,
+			CDEV_MP_NB_OBJS,
+			sess_sz,
+			CDEV_MP_CACHE_SZ,
+			0, NULL, NULL, NULL,
+			NULL, socket_id,
+			0);
+	ctx->session_priv_pool = sess_mp;
+
+	if (ctx->session_priv_pool == NULL)
+		rte_exit(EXIT_FAILURE,
+			"Cannot init session priv pool on socket %d\n",
+			socket_id);
+	else
+		printf("Allocated session priv pool on socket %d\n",
+			socket_id);
+}
+
 static void
 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
 {
@@ -2397,9 +2379,11 @@ main(int32_t argc, char **argv)
 {
 	int32_t ret;
 	uint32_t lcore_id;
+	uint32_t i;
 	uint8_t socket_id;
 	uint16_t portid;
 	uint64_t req_rx_offloads, req_tx_offloads;
+	size_t sess_sz;
 
 	/* init EAL */
 	ret = rte_eal_init(argc, argv);
@@ -2427,7 +2411,8 @@ main(int32_t argc, char **argv)
 
 	nb_lcores = rte_lcore_count();
 
-	/* Replicate each context per socket */
+	sess_sz = max_session_size();
+
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
@@ -2437,20 +2422,14 @@ main(int32_t argc, char **argv)
 		else
 			socket_id = 0;
 
+		/* mbuf_pool is initialised by the pool_init() function*/
 		if (socket_ctx[socket_id].mbuf_pool)
 			continue;
 
-		/* initilaze SPD */
-		sp4_init(&socket_ctx[socket_id], socket_id);
-
-		sp6_init(&socket_ctx[socket_id], socket_id);
-
-		/* initilaze SAD */
-		sa_init(&socket_ctx[socket_id], socket_id);
-
-		rt_init(&socket_ctx[socket_id], socket_id);
-
 		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
+		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
+		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
+			sess_sz);
 	}
 
 	RTE_ETH_FOREACH_DEV(portid) {
@@ -2468,7 +2447,11 @@ main(int32_t argc, char **argv)
 		if ((enabled_port_mask & (1 << portid)) == 0)
 			continue;
 
-		/* Start device */
+		/*
+		 * Start device
+		 * note: device must be started before a flow rule
+		 * can be installed.
+		 */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
 			rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
@@ -2493,6 +2476,19 @@ main(int32_t argc, char **argv)
 			rte_exit(EXIT_FAILURE, "failed at reassemble init");
 	}
 
+	/* Replicate each context per socket */
+	for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
+		socket_id = rte_socket_id_by_idx(i);
+		if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
+			(socket_ctx[socket_id].sa_in == NULL) &&
+			(socket_ctx[socket_id].sa_out == NULL)) {
+			sa_init(&socket_ctx[socket_id], socket_id);
+			sp4_init(&socket_ctx[socket_id], socket_id);
+			sp6_init(&socket_ctx[socket_id], socket_id);
+			rt_init(&socket_ctx[socket_id], socket_id);
+		}
+	}
+
 	check_all_ports_link_status(enabled_port_mask);
 
 	/* launch per-lcore init on every lcore */
diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c
index d1cbdc3..0f23cb1 100644
--- a/examples/ipsec-secgw/ipsec.c
+++ b/examples/ipsec-secgw/ipsec.c
@@ -40,7 +40,7 @@ set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
 }
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 {
 	struct rte_cryptodev_info cdev_info;
 	unsigned long cdev_id_qp = 0;
@@ -53,19 +53,17 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 	key.auth_algo = (uint8_t)sa->auth_algo;
 	key.aead_algo = (uint8_t)sa->aead_algo;
 
-	if (sa->type == RTE_SECURITY_ACTION_TYPE_NONE) {
-		ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
-				(void **)&cdev_id_qp);
-		if (ret < 0) {
-			RTE_LOG(ERR, IPSEC,
+	ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
+			(void **)&cdev_id_qp);
+	if (ret < 0) {
+		RTE_LOG(ERR, IPSEC,
 				"No cryptodev: core %u, cipher_algo %u, "
 				"auth_algo %u, aead_algo %u\n",
 				key.lcore_id,
 				key.cipher_algo,
 				key.auth_algo,
 				key.aead_algo);
-			return -1;
-		}
+		return -1;
 	}
 
 	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
@@ -107,231 +105,268 @@ create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa)
 				"SEC Session init failed: err: %d\n", ret);
 				return -1;
 			}
-		} else if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
-			struct rte_flow_error err;
-			struct rte_security_ctx *ctx = (struct rte_security_ctx *)
-							rte_eth_dev_get_sec_ctx(
-							sa->portid);
-			const struct rte_security_capability *sec_cap;
-			int ret = 0;
-
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_priv_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+		} else {
+			RTE_LOG(ERR, IPSEC, "Inline not supported\n");
+			return -1;
+		}
+	} else {
+		sa->crypto_session = rte_cryptodev_sym_session_create(
+				ipsec_ctx->session_pool);
+		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
+				sa->crypto_session, sa->xforms,
+				ipsec_ctx->session_priv_pool);
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
+		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
+				&cdev_info);
+	}
 
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					sess_conf.ipsec.mode &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+	sa->cdev_id_qp = cdev_id_qp;
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
-				"No suitable security capability found\n");
-				return -1;
-			}
+	return 0;
+}
 
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
-			sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
-
-			if (IS_IP6(sa->flags)) {
-				sa->pattern[1].mask = &rte_flow_item_ipv6_mask;
-				sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
-				sa->pattern[1].spec = &sa->ipv6_spec;
-
-				memcpy(sa->ipv6_spec.hdr.dst_addr,
-					sa->dst.ip.ip6.ip6_b, 16);
-				memcpy(sa->ipv6_spec.hdr.src_addr,
-				       sa->src.ip.ip6.ip6_b, 16);
-			} else if (IS_IP4(sa->flags)) {
-				sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
-				sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
-				sa->pattern[1].spec = &sa->ipv4_spec;
-
-				sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
-				sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
-			}
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa)
+{
+	int32_t ret = 0;
+	struct rte_security_ctx *sec_ctx;
+	struct rte_security_session_conf sess_conf = {
+		.action_type = sa->type,
+		.protocol = RTE_SECURITY_PROTOCOL_IPSEC,
+		{.ipsec = {
+			.spi = sa->spi,
+			.salt = sa->salt,
+			.options = { 0 },
+			.direction = sa->direction,
+			.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
+			.mode = (sa->flags == IP4_TUNNEL ||
+					sa->flags == IP6_TUNNEL) ?
+					RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
+					RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
+		} },
+		.crypto_xform = sa->xforms,
+		.userdata = NULL,
+	};
+
+	RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
+		sa->spi, sa->portid);
+
+	if (sa->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+		struct rte_flow_error err;
+		const struct rte_security_capability *sec_cap;
+		int ret = 0;
+
+		sec_ctx = (struct rte_security_ctx *)
+					rte_eth_dev_get_sec_ctx(
+					sa->portid);
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				" rte_eth_dev_get_sec_ctx failed\n");
+			return -1;
+		}
 
-			sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
-			sa->pattern[2].spec = &sa->esp_spec;
-			sa->pattern[2].mask = &rte_flow_item_esp_mask;
-			sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+		sa->sec_session = rte_security_session_create(sec_ctx,
+				&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"SEC Session init failed: err: %d\n", ret);
+			return -1;
+		}
 
-			sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
-			sa->action[0].conf = sa->sec_session;
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
+				"No suitable security capability found\n");
+			return -1;
+		}
 
-			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
+		sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
+
+		if (IS_IP6(sa->flags)) {
+			sa->pattern[1].mask = &rte_flow_item_ipv6_mask;
+			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
+			sa->pattern[1].spec = &sa->ipv6_spec;
+
+			memcpy(sa->ipv6_spec.hdr.dst_addr,
+				sa->dst.ip.ip6.ip6_b, 16);
+			memcpy(sa->ipv6_spec.hdr.src_addr,
+			       sa->src.ip.ip6.ip6_b, 16);
+		} else if (IS_IP4(sa->flags)) {
+			sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
+			sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+			sa->pattern[1].spec = &sa->ipv4_spec;
+
+			sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
+			sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
+		}
 
-			sa->attr.egress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
-			sa->attr.ingress = (sa->direction ==
-					RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
-			if (sa->attr.ingress) {
-				uint8_t rss_key[40];
-				struct rte_eth_rss_conf rss_conf = {
-					.rss_key = rss_key,
-					.rss_key_len = 40,
-				};
-				struct rte_eth_dev *eth_dev;
-				uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
-				struct rte_flow_action_rss action_rss;
-				unsigned int i;
-				unsigned int j;
-
-				sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
-				/* Try RSS. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
-				sa->action[1].conf = &action_rss;
-				eth_dev = ctx->device;
-				rte_eth_dev_rss_hash_conf_get(sa->portid,
-							      &rss_conf);
-				for (i = 0, j = 0;
-				     i < eth_dev->data->nb_rx_queues; ++i)
-					if (eth_dev->data->rx_queues[i])
-						queue[j++] = i;
-				action_rss = (struct rte_flow_action_rss){
+		sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
+		sa->pattern[2].spec = &sa->esp_spec;
+		sa->pattern[2].mask = &rte_flow_item_esp_mask;
+		sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
+
+		sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+
+		sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
+		sa->action[0].conf = sa->sec_session;
+
+		sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+
+		sa->attr.egress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
+		sa->attr.ingress = (sa->direction ==
+				RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
+		if (sa->attr.ingress) {
+			uint8_t rss_key[40];
+			struct rte_eth_rss_conf rss_conf = {
+				.rss_key = rss_key,
+				.rss_key_len = 40,
+			};
+			struct rte_eth_dev *eth_dev;
+			uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
+			struct rte_flow_action_rss action_rss;
+			unsigned int i;
+			unsigned int j;
+
+			sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
+			/* Try RSS. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
+			sa->action[1].conf = &action_rss;
+			eth_dev = sec_ctx->device;
+			rte_eth_dev_rss_hash_conf_get(sa->portid, &rss_conf);
+			for (i = 0, j = 0;
+			     i < eth_dev->data->nb_rx_queues; ++i)
+				if (eth_dev->data->rx_queues[i])
+					queue[j++] = i;
+
+			action_rss = (struct rte_flow_action_rss){
 					.types = rss_conf.rss_hf,
 					.key_len = rss_conf.rss_key_len,
 					.queue_num = j,
 					.key = rss_key,
 					.queue = queue,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (!ret)
-					goto flow_create;
-				/* Try Queue. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
-				sa->action[1].conf =
-					&(struct rte_flow_action_queue){
-					.index = 0,
-				};
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				/* Try End. */
-				sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
-				sa->action[1].conf = NULL;
-				ret = rte_flow_validate(sa->portid, &sa->attr,
-							sa->pattern, sa->action,
-							&err);
-				if (ret)
-					goto flow_create_failure;
-			} else if (sa->attr.egress &&
-				   (sa->ol_flags &
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (!ret)
+				goto flow_create;
+			/* Try Queue. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
+			sa->action[1].conf =
+				&(struct rte_flow_action_queue){
+				.index = 0,
+			};
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			/* Try End. */
+			sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
+			sa->action[1].conf = NULL;
+			ret = rte_flow_validate(sa->portid, &sa->attr,
+						sa->pattern, sa->action,
+						&err);
+			if (ret)
+				goto flow_create_failure;
+		} else if (sa->attr.egress &&
+			   (sa->ol_flags &
 				    RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
-				sa->action[1].type =
+			sa->action[1].type =
 					RTE_FLOW_ACTION_TYPE_PASSTHRU;
-				sa->action[2].type =
+			sa->action[2].type =
 					RTE_FLOW_ACTION_TYPE_END;
-			}
+		}
 flow_create:
-			sa->flow = rte_flow_create(sa->portid,
+		sa->flow = rte_flow_create(sa->portid,
 				&sa->attr, sa->pattern, sa->action, &err);
-			if (sa->flow == NULL) {
+		if (sa->flow == NULL) {
 flow_create_failure:
-				RTE_LOG(ERR, IPSEC,
-					"Failed to create ipsec flow msg: %s\n",
-					err.message);
-				return -1;
-			}
-		} else if (sa->type ==
-				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
-			struct rte_security_ctx *ctx =
-					(struct rte_security_ctx *)
-					rte_eth_dev_get_sec_ctx(sa->portid);
-			const struct rte_security_capability *sec_cap;
-
-			if (ctx == NULL) {
-				RTE_LOG(ERR, IPSEC,
-				"Ethernet device doesn't have security features registered\n");
-				return -1;
-			}
-
-			/* Set IPsec parameters in conf */
-			set_ipsec_conf(sa, &(sess_conf.ipsec));
+			RTE_LOG(ERR, IPSEC,
+				"Failed to create ipsec flow msg: %s\n",
+				err.message);
+			return -1;
+		}
+	} else if (sa->type ==	RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		const struct rte_security_capability *sec_cap;
 
-			/* Save SA as userdata for the security session. When
-			 * the packet is received, this userdata will be
-			 * retrieved using the metadata from the packet.
-			 *
-			 * The PMD is expected to set similar metadata for other
-			 * operations, like rte_eth_event, which are tied to
-			 * security session. In such cases, the userdata could
-			 * be obtained to uniquely identify the security
-			 * parameters denoted.
-			 */
+		sec_ctx = (struct rte_security_ctx *)
+				rte_eth_dev_get_sec_ctx(sa->portid);
 
-			sess_conf.userdata = (void *) sa;
+		if (sec_ctx == NULL) {
+			RTE_LOG(ERR, IPSEC,
+				"Ethernet device doesn't have security features registered\n");
+			return -1;
+		}
 
-			sa->sec_session = rte_security_session_create(ctx,
-					&sess_conf, ipsec_ctx->session_pool);
-			if (sa->sec_session == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		/* Set IPsec parameters in conf */
+		set_ipsec_conf(sa, &(sess_conf.ipsec));
+
+		/* Save SA as userdata for the security session. When
+		 * the packet is received, this userdata will be
+		 * retrieved using the metadata from the packet.
+		 *
+		 * The PMD is expected to set similar metadata for other
+		 * operations, like rte_eth_event, which are tied to
+		 * security session. In such cases, the userdata could
+		 * be obtained to uniquely identify the security
+		 * parameters denoted.
+		 */
+
+		sess_conf.userdata = (void *) sa;
+
+		sa->sec_session = rte_security_session_create(sec_ctx,
+					&sess_conf, skt_ctx->session_pool);
+		if (sa->sec_session == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"SEC Session init failed: err: %d\n", ret);
-				return -1;
-			}
-
-			sec_cap = rte_security_capabilities_get(ctx);
+			return -1;
+		}
 
-			if (sec_cap == NULL) {
-				RTE_LOG(ERR, IPSEC,
+		sec_cap = rte_security_capabilities_get(sec_ctx);
+		if (sec_cap == NULL) {
+			RTE_LOG(ERR, IPSEC,
 				"No capabilities registered\n");
-				return -1;
-			}
+			return -1;
+		}
 
-			/* iterate until ESP tunnel*/
-			while (sec_cap->action !=
-					RTE_SECURITY_ACTION_TYPE_NONE) {
-
-				if (sec_cap->action == sa->type &&
-				    sec_cap->protocol ==
-					RTE_SECURITY_PROTOCOL_IPSEC &&
-				    sec_cap->ipsec.mode ==
-					sess_conf.ipsec.mode &&
-				    sec_cap->ipsec.direction == sa->direction)
-					break;
-				sec_cap++;
-			}
+		/* iterate until ESP tunnel*/
+		while (sec_cap->action !=
+				RTE_SECURITY_ACTION_TYPE_NONE) {
+			if (sec_cap->action == sa->type &&
+			    sec_cap->protocol ==
+				RTE_SECURITY_PROTOCOL_IPSEC &&
+			    sec_cap->ipsec.mode ==
+				sess_conf.ipsec.mode &&
+			    sec_cap->ipsec.direction == sa->direction)
+				break;
+			sec_cap++;
+		}
 
-			if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
-				RTE_LOG(ERR, IPSEC,
+		if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
+			RTE_LOG(ERR, IPSEC,
 				"No suitable security capability found\n");
-				return -1;
-			}
-
-			sa->ol_flags = sec_cap->ol_flags;
-			sa->security_ctx = ctx;
+			return -1;
 		}
-	} else {
-		sa->crypto_session = rte_cryptodev_sym_session_create(
-				ipsec_ctx->session_pool);
-		rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
-				sa->crypto_session, sa->xforms,
-				ipsec_ctx->session_priv_pool);
 
-		rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
-				&cdev_info);
+		sa->ol_flags = sec_cap->ol_flags;
+		sa->security_ctx = sec_ctx;
 	}
-	sa->cdev_id_qp = cdev_id_qp;
+	sa->cdev_id_qp = 0;
 
 	return 0;
 }
@@ -398,7 +433,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -417,7 +452,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			rte_prefetch0(&priv->sym_cop);
 
 			if ((unlikely(sa->crypto_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
+				create_lookaside_session(ipsec_ctx, sa)) {
 				rte_pktmbuf_free(pkts[i]);
 				continue;
 			}
@@ -432,12 +467,7 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 			}
 			break;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
+			RTE_ASSERT(sa->sec_session != NULL);
 			ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 			if (sa->ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 				rte_security_set_pkt_metadata(
@@ -445,17 +475,11 @@ ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 						sa->sec_session, pkts[i], NULL);
 			continue;
 		case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
+			RTE_ASSERT(sa->sec_session != NULL);
 			priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 			priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 
 			rte_prefetch0(&priv->sym_cop);
-
-			if ((unlikely(sa->sec_session == NULL)) &&
-					create_session(ipsec_ctx, sa)) {
-				rte_pktmbuf_free(pkts[i]);
-				continue;
-			}
-
 			rte_security_attach_session(&priv->cop,
 					sa->sec_session);
 
diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
index 6e48466..1efa6e4 100644
--- a/examples/ipsec-secgw/ipsec.h
+++ b/examples/ipsec-secgw/ipsec.h
@@ -338,6 +338,9 @@ void
 enqueue_cop_burst(struct cdev_qp *cqp);
 
 int
-create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa);
+
+int
+create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa);
 
 #endif /* __IPSEC_H__ */
diff --git a/examples/ipsec-secgw/ipsec_process.c b/examples/ipsec-secgw/ipsec_process.c
index 3f9cacb..868f1a2 100644
--- a/examples/ipsec-secgw/ipsec_process.c
+++ b/examples/ipsec-secgw/ipsec_process.c
@@ -95,22 +95,23 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
 	/* setup crypto section */
 	if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE) {
 		if (sa->crypto_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->crypto.ses = sa->crypto_session;
 	/* setup session action type */
-	} else {
+	} else if (sa->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 		if (sa->sec_session == NULL) {
-			rc = create_session(ctx, sa);
+			rc = create_lookaside_session(ctx, sa);
 			if (rc != 0)
 				return rc;
 		}
 		ss->security.ses = sa->sec_session;
 		ss->security.ctx = sa->security_ctx;
 		ss->security.ol_flags = sa->ol_flags;
-	}
+	} else
+		RTE_ASSERT(0);
 
 	rc = rte_ipsec_session_prepare(ss);
 	if (rc != 0)
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index d700c8e..c3cf3bd 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -868,12 +868,14 @@ sa_add_address_inline_crypto(struct ipsec_sa *sa)
 
 static int
 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries, uint32_t inbound)
+		uint32_t nb_entries, uint32_t inbound,
+		struct socket_ctx *skt_ctx)
 {
 	struct ipsec_sa *sa;
 	uint32_t i, idx;
 	uint16_t iv_length, aad_length;
 	int inline_status;
+	int32_t rc;
 
 	/* for ESN upper 32 bits of SQN also need to be part of AAD */
 	aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
@@ -936,6 +938,17 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 			sa->xforms = &sa_ctx->xf[idx].a;
 
+			if (sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
+				sa->type ==
+				RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
+				rc = create_inline_session(skt_ctx, sa);
+				if (rc != 0) {
+					RTE_LOG(ERR, IPSEC_ESP,
+						"create_inline_session() failed\n");
+					return -EINVAL;
+				}
+			}
 			print_one_sa_rule(sa, inbound);
 		} else {
 			switch (sa->cipher_algo) {
@@ -1011,16 +1024,16 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 
 static inline int
 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 0);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
 }
 
 static inline int
 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
-		uint32_t nb_entries)
+		uint32_t nb_entries, struct socket_ctx *skt_ctx)
 {
-	return sa_add_rules(sa_ctx, entries, nb_entries, 1);
+	return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
 }
 
 /*
@@ -1086,10 +1099,12 @@ fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
 	return 0;
 }
 
-static void
+static int
 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 	const struct ipsec_sa *lsa)
 {
+	int32_t rc = 0;
+
 	ss->sa = sa;
 	ss->type = lsa->type;
 
@@ -1102,6 +1117,17 @@ fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa,
 		ss->security.ctx = lsa->security_ctx;
 		ss->security.ol_flags = lsa->ol_flags;
 	}
+
+	if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
+		ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
+		if (ss->security.ses != NULL) {
+			rc = rte_ipsec_session_prepare(ss);
+			if (rc != 0)
+				memset(ss, 0, sizeof(*ss));
+		}
+	}
+
+	return rc;
 }
 
 /*
@@ -1136,8 +1162,8 @@ ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
 	if (rc < 0)
 		return rc;
 
-	fill_ipsec_session(&lsa->ips, sa, lsa);
-	return 0;
+	rc = fill_ipsec_session(&lsa->ips, sa, lsa);
+	return rc;
 }
 
 /*
@@ -1240,7 +1266,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in);
+		sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
@@ -1260,7 +1286,7 @@ sa_init(struct socket_ctx *ctx, int32_t socket_id)
 				"context %s in socket %d\n", rte_errno,
 				name, socket_id);
 
-		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out);
+		sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
 
 		if (app_sa_prm.enable != 0) {
 			rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
-- 
2.7.4

[dpdk-dev] [PATCH v7 2/2] examples/ipsec-secgw/test: fix inline test scripts

[Bernard Iremonger]

Remove workaround in tun_aesgcm_defs.sh and trs_aesgcm_defs.sh
to get around the bug where the first inbound packet is dropped
for inline crypto.

Fixes: 929784452094 ("examples/ipsec-secgw: add scripts for functional test")
Cc: stable@dpdk.org

Signed-off-by: Bernard Iremonger <bernard.iremonger@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
---
 examples/ipsec-secgw/test/trs_aesgcm_defs.sh | 10 ----------
 examples/ipsec-secgw/test/tun_aesgcm_defs.sh | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
index a4d902b..8382d3d 100755
--- a/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/trs_aesgcm_defs.sh
@@ -33,11 +33,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -68,9 +63,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV6}
 }
diff --git a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
index 1764ef6..8ae6532 100755
--- a/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
+++ b/examples/ipsec-secgw/test/tun_aesgcm_defs.sh
@@ -35,11 +35,6 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping -c 1 ${LOCAL_IPV4}
 }
 
 config6_remote_xfrm()
@@ -72,9 +67,4 @@ aead "rfc4106\(gcm\(aes\)\)" \
 
 	ssh ${REMOTE_HOST} ip xfrm policy list
 	ssh ${REMOTE_HOST} ip xfrm state list
-
-	# to overcome problem with ipsec-secgw for inline mode,
-	# when first packet(s) will be always dropped.
-	# note that ping will fail here
-	ssh ${REMOTE_HOST} ping6 -c 1 ${LOCAL_IPV6}
 }
-- 
2.7.4

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Akhil Goyal]

> -----Original Message-----
> From: Bernard Iremonger <bernard.iremonger@intel.com>
> Sent: Wednesday, July 10, 2019 4:53 PM
> To: dev@dpdk.org; konstantin.ananyev@intel.com; Akhil Goyal
> <akhil.goyal@nxp.com>
> Cc: Bernard Iremonger <bernard.iremonger@intel.com>
> Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> 
> This patchset fixes the issue of the first inbound packet
> being dropped for inline crypto.
> 
> Changes in v7:
> --------------
> Rebased to DPDK 19.08-rc1
> 
> Changes in v6:
> --------------
> Rebased to latest master.
> Minor changes to the following functions in ipsec.c:
> create_lookaside_session()
> create_inline_session()
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
>  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 ++-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 403 insertions(+), 370 deletions(-)
> 
> Changes in v5:
> -------------
> The v2 patchset has been rebased to the latest master.
> The v4 patchset has been dropped as it caused issues with the lookaside code
> which we are unable to test.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++--------------
>  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 +++--
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
>  7 files changed, 245 insertions(+), 201 deletions(-)
> 
> Changes in v2:
> -------------
> The first three patches of the v1 have been squashed.
> The commit message for the squashed patch has been updated.
> Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> A patch to fix the test scripts has been added.
> 
> Bernard Iremonger (2):
>   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
>   examples/ipsec-secgw/test: fix inline test scripts
> 
>  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
>  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
>  examples/ipsec-secgw/ipsec.h                 |   5 +-
>  examples/ipsec-secgw/ipsec_process.c         |   9 +-
>  examples/ipsec-secgw/sa.c                    |  46 ++-
>  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
>  examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
>  7 files changed, 405 insertions(+), 375 deletions(-)
> 
> --
> 2.7.4

It was rebased manually, so please verify.
Applied to dpdk-next-crypto

Thanks.

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>

> > Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> >
> > This patchset fixes the issue of the first inbound packet being
> > dropped for inline crypto.
> >
> > Changes in v7:
> > --------------
> > Rebased to DPDK 19.08-rc1
> >
> > Changes in v6:
> > --------------
> > Rebased to latest master.
> > Minor changes to the following functions in ipsec.c:
> > create_lookaside_session()
> > create_inline_session()
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
> >  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++-------------
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 ++-
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> >  7 files changed, 403 insertions(+), 370 deletions(-)
> >
> > Changes in v5:
> > -------------
> > The v2 patchset has been rebased to the latest master.
> > The v4 patchset has been dropped as it caused issues with the
> > lookaside code which we are unable to test.
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++-----------
> ---
> >  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 +++--
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
> >  7 files changed, 245 insertions(+), 201 deletions(-)
> >
> > Changes in v2:
> > -------------
> > The first three patches of the v1 have been squashed.
> > The commit message for the squashed patch has been updated.
> > Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> > A patch to fix the test scripts has been added.
> >
> > Bernard Iremonger (2):
> >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> >   examples/ipsec-secgw/test: fix inline test scripts
> >
> >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
> >  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++-------------
> >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> >  examples/ipsec-secgw/sa.c                    |  46 ++-
> >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> >  7 files changed, 405 insertions(+), 375 deletions(-)
> >
> > --
> > 2.7.4
> 
> It was rebased manually, so please verify.
> Applied to dpdk-next-crypto
> 
> Thanks.

I will verify.

Regards,

Bernard.

Re: [dpdk-dev] [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped

[Iremonger, Bernard]

Hi Akhil,

<snip>

> > > Subject: [PATCH v7 0/2] examples/ipsec-secgw: fix 1st pkt dropped
> > >
> > > This patchset fixes the issue of the first inbound packet being
> > > dropped for inline crypto.
> > >
> > > Changes in v7:
> > > --------------
> > > Rebased to DPDK 19.08-rc1
> > >
> > > Changes in v6:
> > > --------------
> > > Rebased to latest master.
> > > Minor changes to the following functions in ipsec.c:
> > > create_lookaside_session()
> > > create_inline_session()
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++--------
> > >  examples/ipsec-secgw/ipsec.c                 | 449 ++++++++++++++------------
> -
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 ++-
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> > >  7 files changed, 403 insertions(+), 370 deletions(-)
> > >
> > > Changes in v5:
> > > -------------
> > > The v2 patchset has been rebased to the latest master.
> > > The v4 patchset has been dropped as it caused issues with the
> > > lookaside code which we are unable to test.
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++++++++---------
> --
> > ---
> > >  examples/ipsec-secgw/ipsec.c                 | 122 +++++++++-----
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 +++--
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 --
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 --
> > >  7 files changed, 245 insertions(+), 201 deletions(-)
> > >
> > > Changes in v2:
> > > -------------
> > > The first three patches of the v1 have been squashed.
> > > The commit message for the squashed patch has been updated.
> > > Patches 4,5 and 6 of the v1 have been dropped from this patchset.
> > > A patch to fix the test scripts has been added.
> > >
> > > Bernard Iremonger (2):
> > >   examples/ipsec-secgw: fix 1st pkt dropped for inline crypto
> > >   examples/ipsec-secgw/test: fix inline test scripts
> > >
> > >  examples/ipsec-secgw/ipsec-secgw.c           | 244 +++++++-------
> > >  examples/ipsec-secgw/ipsec.c                 | 456 ++++++++++++++------------
> -
> > >  examples/ipsec-secgw/ipsec.h                 |   5 +-
> > >  examples/ipsec-secgw/ipsec_process.c         |   9 +-
> > >  examples/ipsec-secgw/sa.c                    |  46 ++-
> > >  examples/ipsec-secgw/test/trs_aesgcm_defs.sh |  10 -
> > > examples/ipsec-secgw/test/tun_aesgcm_defs.sh |  10 -
> > >  7 files changed, 405 insertions(+), 375 deletions(-)
> > >
> > > --
> > > 2.7.4
> >
> > It was rebased manually, so please verify.
> > Applied to dpdk-next-crypto
> >
> > Thanks.
> 
> I will verify.
> 
> Regards,
> 
> Bernard.
> 
> 
Our tests are passing fine with the rebased patches.

Regards,

Bernard.