From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 50DBDA00E6 for ; Wed, 20 Mar 2019 19:48:10 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 42EDD1B3E0; Wed, 20 Mar 2019 19:47:40 +0100 (CET) Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by dpdk.org (Postfix) with ESMTP id 705681B3A5 for ; Wed, 20 Mar 2019 19:47:34 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 20 Mar 2019 11:47:33 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,249,1549958400"; d="scan'208";a="308891841" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by orsmga005.jf.intel.com with ESMTP; 20 Mar 2019 11:47:31 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: akhil.goyal@nxp.com, olivier.matz@6wind.com, Konstantin Ananyev Date: Wed, 20 Mar 2019 18:46:53 +0000 Message-Id: <20190320184655.17004-6-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20190320184655.17004-1-konstantin.ananyev@intel.com> References: <1551381661-21078-1-git-send-email-konstantin.ananyev@intel.com> <20190320184655.17004-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 5/7] ipsec: move inbound and outbound code into different files X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Content-Type: text/plain; charset="UTF-8" Message-ID: <20190320184653.n3Y8P_Jcul_As3-SSwyOo6_YIQfv4ByMT713re7tc1c@z> sa.c becomes too big, so decided to split it into 3 chunks: - sa.c - control path related functions (init/fini, etc.) - esp_inb.c - ESP inbound packet processing - esp_outb.c - ESP outbound packet processing Plus few changes in internal function names to follow the same code convention. No functional changes introduced. Signed-off-by: Konstantin Ananyev --- lib/librte_ipsec/Makefile | 2 + lib/librte_ipsec/crypto.h | 17 + lib/librte_ipsec/esp_inb.c | 408 ++++++++++++++ lib/librte_ipsec/esp_outb.c | 535 ++++++++++++++++++ lib/librte_ipsec/ipsec_sqn.h | 30 - lib/librte_ipsec/meson.build | 2 +- lib/librte_ipsec/misc.h | 41 ++ lib/librte_ipsec/sa.c | 1011 ++-------------------------------- lib/librte_ipsec/sa.h | 40 ++ 9 files changed, 1086 insertions(+), 1000 deletions(-) create mode 100644 lib/librte_ipsec/esp_inb.c create mode 100644 lib/librte_ipsec/esp_outb.c create mode 100644 lib/librte_ipsec/misc.h diff --git a/lib/librte_ipsec/Makefile b/lib/librte_ipsec/Makefile index 77506d6ad..e80926baa 100644 --- a/lib/librte_ipsec/Makefile +++ b/lib/librte_ipsec/Makefile @@ -16,6 +16,8 @@ EXPORT_MAP := rte_ipsec_version.map LIBABIVER := 1 # all source are stored in SRCS-y +SRCS-$(CONFIG_RTE_LIBRTE_IPSEC) += esp_inb.c +SRCS-$(CONFIG_RTE_LIBRTE_IPSEC) += esp_outb.c SRCS-$(CONFIG_RTE_LIBRTE_IPSEC) += sa.c SRCS-$(CONFIG_RTE_LIBRTE_IPSEC) += ses.c diff --git a/lib/librte_ipsec/crypto.h b/lib/librte_ipsec/crypto.h index b5f264831..5a7b11f56 100644 --- a/lib/librte_ipsec/crypto.h +++ b/lib/librte_ipsec/crypto.h @@ -145,4 +145,21 @@ remove_sqh(void *picv, uint32_t icv_len) icv[i] = icv[i + 1]; } +/* + * setup crypto ops for LOOKASIDE_NONE (pure crypto) type of devices. + */ +static inline void +lksd_none_cop_prepare(struct rte_crypto_op *cop, + struct rte_cryptodev_sym_session *cs, struct rte_mbuf *mb) +{ + struct rte_crypto_sym_op *sop; + + sop = cop->sym; + cop->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; + cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + cop->sess_type = RTE_CRYPTO_OP_WITH_SESSION; + sop->m_src = mb; + __rte_crypto_sym_op_attach_sym_session(sop, cs); +} + #endif /* _CRYPTO_H_ */ diff --git a/lib/librte_ipsec/esp_inb.c b/lib/librte_ipsec/esp_inb.c new file mode 100644 index 000000000..562185ebe --- /dev/null +++ b/lib/librte_ipsec/esp_inb.c @@ -0,0 +1,408 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "sa.h" +#include "ipsec_sqn.h" +#include "crypto.h" +#include "iph.h" +#include "misc.h" +#include "pad.h" + +/* + * setup crypto op and crypto sym op for ESP inbound tunnel packet. + */ +static inline int32_t +inb_cop_prepare(struct rte_crypto_op *cop, + const struct rte_ipsec_sa *sa, struct rte_mbuf *mb, + const union sym_op_data *icv, uint32_t pofs, uint32_t plen) +{ + struct rte_crypto_sym_op *sop; + struct aead_gcm_iv *gcm; + uint64_t *ivc, *ivp; + uint32_t clen; + + clen = plen - sa->ctp.cipher.length; + if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0) + return -EINVAL; + + /* fill sym op fields */ + sop = cop->sym; + + /* AEAD (AES_GCM) case */ + if (sa->aad_len != 0) { + sop->aead.data.offset = pofs + sa->ctp.cipher.offset; + sop->aead.data.length = clen; + sop->aead.digest.data = icv->va; + sop->aead.digest.phys_addr = icv->pa; + sop->aead.aad.data = icv->va + sa->icv_len; + sop->aead.aad.phys_addr = icv->pa + sa->icv_len; + + /* fill AAD IV (located inside crypto op) */ + gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *, + sa->iv_ofs); + ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *, + pofs + sizeof(struct esp_hdr)); + aead_gcm_iv_fill(gcm, ivp[0], sa->salt); + /* CRYPT+AUTH case */ + } else { + sop->cipher.data.offset = pofs + sa->ctp.cipher.offset; + sop->cipher.data.length = clen; + sop->auth.data.offset = pofs + sa->ctp.auth.offset; + sop->auth.data.length = plen - sa->ctp.auth.length; + sop->auth.digest.data = icv->va; + sop->auth.digest.phys_addr = icv->pa; + + /* copy iv from the input packet to the cop */ + ivc = rte_crypto_op_ctod_offset(cop, uint64_t *, sa->iv_ofs); + ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *, + pofs + sizeof(struct esp_hdr)); + copy_iv(ivc, ivp, sa->iv_len); + } + return 0; +} + +/* + * for pure cryptodev (lookaside none) depending on SA settings, + * we might have to write some extra data to the packet. + */ +static inline void +inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, + const union sym_op_data *icv) +{ + struct aead_gcm_aad *aad; + + /* insert SQN.hi between ESP trailer and ICV */ + if (sa->sqh_len != 0) + insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len); + + /* + * fill AAD fields, if any (aad fields are placed after icv), + * right now we support only one AEAD algorithm: AES-GCM. + */ + if (sa->aad_len != 0) { + aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); + aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); + } +} + +/* + * setup/update packet data and metadata for ESP inbound tunnel case. + */ +static inline int32_t +inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn, + struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv) +{ + int32_t rc; + uint64_t sqn; + uint32_t icv_ofs, plen; + struct rte_mbuf *ml; + struct esp_hdr *esph; + + esph = rte_pktmbuf_mtod_offset(mb, struct esp_hdr *, hlen); + + /* + * retrieve and reconstruct SQN, then check it, then + * convert it back into network byte order. + */ + sqn = rte_be_to_cpu_32(esph->seq); + if (IS_ESN(sa)) + sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz); + + rc = esn_inb_check_sqn(rsn, sa, sqn); + if (rc != 0) + return rc; + + sqn = rte_cpu_to_be_64(sqn); + + /* start packet manipulation */ + plen = mb->pkt_len; + plen = plen - hlen; + + ml = rte_pktmbuf_lastseg(mb); + icv_ofs = ml->data_len - sa->icv_len + sa->sqh_len; + + /* we have to allocate space for AAD somewhere, + * right now - just use free trailing space at the last segment. + * Would probably be more convenient to reserve space for AAD + * inside rte_crypto_op itself + * (again for IV space is already reserved inside cop). + */ + if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml)) + return -ENOSPC; + + icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs); + icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs); + + inb_pkt_xprepare(sa, sqn, icv); + return plen; +} + +/* + * setup/update packets and crypto ops for ESP inbound case. + */ +uint16_t +esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num) +{ + int32_t rc; + uint32_t i, k, hl; + struct rte_ipsec_sa *sa; + struct rte_cryptodev_sym_session *cs; + struct replay_sqn *rsn; + union sym_op_data icv; + uint32_t dr[num]; + + sa = ss->sa; + cs = ss->crypto.ses; + rsn = rsn_acquire(sa); + + k = 0; + for (i = 0; i != num; i++) { + + hl = mb[i]->l2_len + mb[i]->l3_len; + rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv); + if (rc >= 0) { + lksd_none_cop_prepare(cop[k], cs, mb[i]); + rc = inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc); + } + + k += (rc == 0); + if (rc != 0) { + dr[i - k] = i; + rte_errno = -rc; + } + } + + rsn_release(sa, rsn); + + /* copy not prepared mbufs beyond good ones */ + if (k != num && k != 0) + mbuf_bad_move(mb, dr, num, num - k); + + return k; +} + +/* + * process ESP inbound tunnel packet. + */ +static inline int +inb_tun_single_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb, + uint32_t *sqn) +{ + uint32_t hlen, icv_len, tlen; + struct esp_hdr *esph; + struct esp_tail *espt; + struct rte_mbuf *ml; + char *pd; + + if (mb->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) + return -EBADMSG; + + icv_len = sa->icv_len; + + ml = rte_pktmbuf_lastseg(mb); + espt = rte_pktmbuf_mtod_offset(ml, struct esp_tail *, + ml->data_len - icv_len - sizeof(*espt)); + + /* + * check padding and next proto. + * return an error if something is wrong. + */ + pd = (char *)espt - espt->pad_len; + if (espt->next_proto != sa->proto || + memcmp(pd, esp_pad_bytes, espt->pad_len)) + return -EINVAL; + + /* cut of ICV, ESP tail and padding bytes */ + tlen = icv_len + sizeof(*espt) + espt->pad_len; + ml->data_len -= tlen; + mb->pkt_len -= tlen; + + /* cut of L2/L3 headers, ESP header and IV */ + hlen = mb->l2_len + mb->l3_len; + esph = rte_pktmbuf_mtod_offset(mb, struct esp_hdr *, hlen); + rte_pktmbuf_adj(mb, hlen + sa->ctp.cipher.offset); + + /* retrieve SQN for later check */ + *sqn = rte_be_to_cpu_32(esph->seq); + + /* reset mbuf metatdata: L2/L3 len, packet type */ + mb->packet_type = RTE_PTYPE_UNKNOWN; + mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) | + sa->tx_offload.val; + + /* clear the PKT_RX_SEC_OFFLOAD flag if set */ + mb->ol_flags &= ~(mb->ol_flags & PKT_RX_SEC_OFFLOAD); + return 0; +} + +/* + * process ESP inbound transport packet. + */ +static inline int +inb_trs_single_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb, + uint32_t *sqn) +{ + uint32_t hlen, icv_len, l2len, l3len, tlen; + struct esp_hdr *esph; + struct esp_tail *espt; + struct rte_mbuf *ml; + char *np, *op, *pd; + + if (mb->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) + return -EBADMSG; + + icv_len = sa->icv_len; + + ml = rte_pktmbuf_lastseg(mb); + espt = rte_pktmbuf_mtod_offset(ml, struct esp_tail *, + ml->data_len - icv_len - sizeof(*espt)); + + /* check padding, return an error if something is wrong. */ + pd = (char *)espt - espt->pad_len; + if (memcmp(pd, esp_pad_bytes, espt->pad_len)) + return -EINVAL; + + /* cut of ICV, ESP tail and padding bytes */ + tlen = icv_len + sizeof(*espt) + espt->pad_len; + ml->data_len -= tlen; + mb->pkt_len -= tlen; + + /* retrieve SQN for later check */ + l2len = mb->l2_len; + l3len = mb->l3_len; + hlen = l2len + l3len; + op = rte_pktmbuf_mtod(mb, char *); + esph = (struct esp_hdr *)(op + hlen); + *sqn = rte_be_to_cpu_32(esph->seq); + + /* cut off ESP header and IV, update L3 header */ + np = rte_pktmbuf_adj(mb, sa->ctp.cipher.offset); + remove_esph(np, op, hlen); + update_trs_l3hdr(sa, np + l2len, mb->pkt_len, l2len, l3len, + espt->next_proto); + + /* reset mbuf packet type */ + mb->packet_type &= (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); + + /* clear the PKT_RX_SEC_OFFLOAD flag if set */ + mb->ol_flags &= ~(mb->ol_flags & PKT_RX_SEC_OFFLOAD); + return 0; +} + +/* + * for group of ESP inbound packets perform SQN check and update. + */ +static inline uint16_t +esp_inb_rsn_update(struct rte_ipsec_sa *sa, const uint32_t sqn[], + uint32_t dr[], uint16_t num) +{ + uint32_t i, k; + struct replay_sqn *rsn; + + rsn = rsn_update_start(sa); + + k = 0; + for (i = 0; i != num; i++) { + if (esn_inb_update_sqn(rsn, sa, sqn[i]) == 0) + k++; + else + dr[i - k] = i; + } + + rsn_update_finish(sa, rsn); + return k; +} + +/* + * process group of ESP inbound tunnel packets. + */ +uint16_t +esp_inb_tun_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + uint32_t i, k, n; + struct rte_ipsec_sa *sa; + uint32_t sqn[num]; + uint32_t dr[num]; + + sa = ss->sa; + + /* process packets, extract seq numbers */ + + k = 0; + for (i = 0; i != num; i++) { + /* good packet */ + if (inb_tun_single_pkt_process(sa, mb[i], sqn + k) == 0) + k++; + /* bad packet, will drop from furhter processing */ + else + dr[i - k] = i; + } + + /* handle unprocessed mbufs */ + if (k != num && k != 0) + mbuf_bad_move(mb, dr, num, num - k); + + /* update SQN and replay winow */ + n = esp_inb_rsn_update(sa, sqn, dr, k); + + /* handle mbufs with wrong SQN */ + if (n != k && n != 0) + mbuf_bad_move(mb, dr, k, k - n); + + if (n != num) + rte_errno = EBADMSG; + + return n; +} + +/* + * process group of ESP inbound transport packets. + */ +uint16_t +esp_inb_trs_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + uint32_t i, k, n; + uint32_t sqn[num]; + struct rte_ipsec_sa *sa; + uint32_t dr[num]; + + sa = ss->sa; + + /* process packets, extract seq numbers */ + + k = 0; + for (i = 0; i != num; i++) { + /* good packet */ + if (inb_trs_single_pkt_process(sa, mb[i], sqn + k) == 0) + k++; + /* bad packet, will drop from furhter processing */ + else + dr[i - k] = i; + } + + /* handle unprocessed mbufs */ + if (k != num && k != 0) + mbuf_bad_move(mb, dr, num, num - k); + + /* update SQN and replay winow */ + n = esp_inb_rsn_update(sa, sqn, dr, k); + + /* handle mbufs with wrong SQN */ + if (n != k && n != 0) + mbuf_bad_move(mb, dr, k, k - n); + + if (n != num) + rte_errno = EBADMSG; + + return n; +} diff --git a/lib/librte_ipsec/esp_outb.c b/lib/librte_ipsec/esp_outb.c new file mode 100644 index 000000000..dce0fe051 --- /dev/null +++ b/lib/librte_ipsec/esp_outb.c @@ -0,0 +1,535 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include "sa.h" +#include "ipsec_sqn.h" +#include "crypto.h" +#include "iph.h" +#include "misc.h" +#include "pad.h" + +/* + * setup crypto op and crypto sym op for ESP outbound packet. + */ +static inline void +outb_cop_prepare(struct rte_crypto_op *cop, + const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD], + const union sym_op_data *icv, uint32_t hlen, uint32_t plen) +{ + struct rte_crypto_sym_op *sop; + struct aead_gcm_iv *gcm; + + /* fill sym op fields */ + sop = cop->sym; + + /* AEAD (AES_GCM) case */ + if (sa->aad_len != 0) { + sop->aead.data.offset = sa->ctp.cipher.offset + hlen; + sop->aead.data.length = sa->ctp.cipher.length + plen; + sop->aead.digest.data = icv->va; + sop->aead.digest.phys_addr = icv->pa; + sop->aead.aad.data = icv->va + sa->icv_len; + sop->aead.aad.phys_addr = icv->pa + sa->icv_len; + + /* fill AAD IV (located inside crypto op) */ + gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *, + sa->iv_ofs); + aead_gcm_iv_fill(gcm, ivp[0], sa->salt); + /* CRYPT+AUTH case */ + } else { + sop->cipher.data.offset = sa->ctp.cipher.offset + hlen; + sop->cipher.data.length = sa->ctp.cipher.length + plen; + sop->auth.data.offset = sa->ctp.auth.offset + hlen; + sop->auth.data.length = sa->ctp.auth.length + plen; + sop->auth.digest.data = icv->va; + sop->auth.digest.phys_addr = icv->pa; + } +} + +/* + * setup/update packet data and metadata for ESP outbound tunnel case. + */ +static inline int32_t +outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, + const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, + union sym_op_data *icv) +{ + uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen; + struct rte_mbuf *ml; + struct esp_hdr *esph; + struct esp_tail *espt; + char *ph, *pt; + uint64_t *iv; + + /* calculate extra header space required */ + hlen = sa->hdr_len + sa->iv_len + sizeof(*esph); + + /* size of ipsec protected data */ + l2len = mb->l2_len; + plen = mb->pkt_len - l2len; + + /* number of bytes to encrypt */ + clen = plen + sizeof(*espt); + clen = RTE_ALIGN_CEIL(clen, sa->pad_align); + + /* pad length + esp tail */ + pdlen = clen - plen; + tlen = pdlen + sa->icv_len; + + /* do append and prepend */ + ml = rte_pktmbuf_lastseg(mb); + if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml)) + return -ENOSPC; + + /* prepend header */ + ph = rte_pktmbuf_prepend(mb, hlen - l2len); + if (ph == NULL) + return -ENOSPC; + + /* append tail */ + pdofs = ml->data_len; + ml->data_len += tlen; + mb->pkt_len += tlen; + pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs); + + /* update pkt l2/l3 len */ + mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) | + sa->tx_offload.val; + + /* copy tunnel pkt header */ + rte_memcpy(ph, sa->hdr, sa->hdr_len); + + /* update original and new ip header fields */ + update_tun_l3hdr(sa, ph + sa->hdr_l3_off, mb->pkt_len, sa->hdr_l3_off, + sqn_low16(sqc)); + + /* update spi, seqn and iv */ + esph = (struct esp_hdr *)(ph + sa->hdr_len); + iv = (uint64_t *)(esph + 1); + copy_iv(iv, ivp, sa->iv_len); + + esph->spi = sa->spi; + esph->seq = sqn_low32(sqc); + + /* offset for ICV */ + pdofs += pdlen + sa->sqh_len; + + /* pad length */ + pdlen -= sizeof(*espt); + + /* copy padding data */ + rte_memcpy(pt, esp_pad_bytes, pdlen); + + /* update esp trailer */ + espt = (struct esp_tail *)(pt + pdlen); + espt->pad_len = pdlen; + espt->next_proto = sa->proto; + + icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); + icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); + + return clen; +} + +/* + * for pure cryptodev (lookaside none) depending on SA settings, + * we might have to write some extra data to the packet. + */ +static inline void +outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, + const union sym_op_data *icv) +{ + uint32_t *psqh; + struct aead_gcm_aad *aad; + + /* insert SQN.hi between ESP trailer and ICV */ + if (sa->sqh_len != 0) { + psqh = (uint32_t *)(icv->va - sa->sqh_len); + psqh[0] = sqn_hi32(sqc); + } + + /* + * fill IV and AAD fields, if any (aad fields are placed after icv), + * right now we support only one AEAD algorithm: AES-GCM . + */ + if (sa->aad_len != 0) { + aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); + aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); + } +} + +/* + * setup/update packets and crypto ops for ESP outbound tunnel case. + */ +uint16_t +esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num) +{ + int32_t rc; + uint32_t i, k, n; + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + struct rte_cryptodev_sym_session *cs; + union sym_op_data icv; + uint64_t iv[IPSEC_MAX_IV_QWORD]; + uint32_t dr[num]; + + sa = ss->sa; + cs = ss->crypto.ses; + + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + k = 0; + for (i = 0; i != n; i++) { + + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv, sqc); + + /* try to update the packet itself */ + rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv); + + /* success, setup crypto op */ + if (rc >= 0) { + outb_pkt_xprepare(sa, sqc, &icv); + lksd_none_cop_prepare(cop[k], cs, mb[i]); + outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc); + k++; + /* failure, put packet into the death-row */ + } else { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not prepared mbufs beyond good ones */ + if (k != n && k != 0) + mbuf_bad_move(mb, dr, n, n - k); + + return k; +} + +/* + * setup/update packet data and metadata for ESP outbound transport case. + */ +static inline int32_t +outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, + const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, + uint32_t l2len, uint32_t l3len, union sym_op_data *icv) +{ + uint8_t np; + uint32_t clen, hlen, pdlen, pdofs, plen, tlen, uhlen; + struct rte_mbuf *ml; + struct esp_hdr *esph; + struct esp_tail *espt; + char *ph, *pt; + uint64_t *iv; + + uhlen = l2len + l3len; + plen = mb->pkt_len - uhlen; + + /* calculate extra header space required */ + hlen = sa->iv_len + sizeof(*esph); + + /* number of bytes to encrypt */ + clen = plen + sizeof(*espt); + clen = RTE_ALIGN_CEIL(clen, sa->pad_align); + + /* pad length + esp tail */ + pdlen = clen - plen; + tlen = pdlen + sa->icv_len; + + /* do append and insert */ + ml = rte_pktmbuf_lastseg(mb); + if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml)) + return -ENOSPC; + + /* prepend space for ESP header */ + ph = rte_pktmbuf_prepend(mb, hlen); + if (ph == NULL) + return -ENOSPC; + + /* append tail */ + pdofs = ml->data_len; + ml->data_len += tlen; + mb->pkt_len += tlen; + pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs); + + /* shift L2/L3 headers */ + insert_esph(ph, ph + hlen, uhlen); + + /* update ip header fields */ + np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len, l2len, l3len, + IPPROTO_ESP); + + /* update spi, seqn and iv */ + esph = (struct esp_hdr *)(ph + uhlen); + iv = (uint64_t *)(esph + 1); + copy_iv(iv, ivp, sa->iv_len); + + esph->spi = sa->spi; + esph->seq = sqn_low32(sqc); + + /* offset for ICV */ + pdofs += pdlen + sa->sqh_len; + + /* pad length */ + pdlen -= sizeof(*espt); + + /* copy padding data */ + rte_memcpy(pt, esp_pad_bytes, pdlen); + + /* update esp trailer */ + espt = (struct esp_tail *)(pt + pdlen); + espt->pad_len = pdlen; + espt->next_proto = np; + + icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); + icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); + + return clen; +} + +/* + * setup/update packets and crypto ops for ESP outbound transport case. + */ +uint16_t +esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num) +{ + int32_t rc; + uint32_t i, k, n, l2, l3; + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + struct rte_cryptodev_sym_session *cs; + union sym_op_data icv; + uint64_t iv[IPSEC_MAX_IV_QWORD]; + uint32_t dr[num]; + + sa = ss->sa; + cs = ss->crypto.ses; + + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + k = 0; + for (i = 0; i != n; i++) { + + l2 = mb[i]->l2_len; + l3 = mb[i]->l3_len; + + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv, sqc); + + /* try to update the packet itself */ + rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3, &icv); + + /* success, setup crypto op */ + if (rc >= 0) { + outb_pkt_xprepare(sa, sqc, &icv); + lksd_none_cop_prepare(cop[k], cs, mb[i]); + outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc); + k++; + /* failure, put packet into the death-row */ + } else { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not prepared mbufs beyond good ones */ + if (k != n && k != 0) + mbuf_bad_move(mb, dr, n, n - k); + + return k; +} + +/* + * process outbound packets for SA with ESN support, + * for algorithms that require SQN.hibits to be implictly included + * into digest computation. + * In that case we have to move ICV bytes back to their proper place. + */ +uint16_t +esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + uint16_t num) +{ + uint32_t i, k, icv_len, *icv; + struct rte_mbuf *ml; + struct rte_ipsec_sa *sa; + uint32_t dr[num]; + + sa = ss->sa; + + k = 0; + icv_len = sa->icv_len; + + for (i = 0; i != num; i++) { + if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) { + ml = rte_pktmbuf_lastseg(mb[i]); + icv = rte_pktmbuf_mtod_offset(ml, void *, + ml->data_len - icv_len); + remove_sqh(icv, icv_len); + k++; + } else + dr[i - k] = i; + } + + /* handle unprocessed mbufs */ + if (k != num) { + rte_errno = EBADMSG; + if (k != 0) + mbuf_bad_move(mb, dr, num, num - k); + } + + return k; +} + +/* + * prepare packets for inline ipsec processing: + * set ol_flags and attach metadata. + */ +static inline void +inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + uint32_t i, ol_flags; + + ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA; + for (i = 0; i != num; i++) { + + mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD; + if (ol_flags != 0) + rte_security_set_pkt_metadata(ss->security.ctx, + ss->security.ses, mb[i], NULL); + } +} + +/* + * process group of ESP outbound tunnel packets destined for + * INLINE_CRYPTO type of device. + */ +uint16_t +inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + int32_t rc; + uint32_t i, k, n; + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + union sym_op_data icv; + uint64_t iv[IPSEC_MAX_IV_QWORD]; + uint32_t dr[num]; + + sa = ss->sa; + + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + k = 0; + for (i = 0; i != n; i++) { + + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv, sqc); + + /* try to update the packet itself */ + rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv); + + k += (rc >= 0); + + /* failure, put packet into the death-row */ + if (rc < 0) { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not processed mbufs beyond good ones */ + if (k != n && k != 0) + mbuf_bad_move(mb, dr, n, n - k); + + inline_outb_mbuf_prepare(ss, mb, k); + return k; +} + +/* + * process group of ESP outbound transport packets destined for + * INLINE_CRYPTO type of device. + */ +uint16_t +inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + int32_t rc; + uint32_t i, k, n, l2, l3; + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + union sym_op_data icv; + uint64_t iv[IPSEC_MAX_IV_QWORD]; + uint32_t dr[num]; + + sa = ss->sa; + + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + k = 0; + for (i = 0; i != n; i++) { + + l2 = mb[i]->l2_len; + l3 = mb[i]->l3_len; + + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv, sqc); + + /* try to update the packet itself */ + rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], + l2, l3, &icv); + + k += (rc >= 0); + + /* failure, put packet into the death-row */ + if (rc < 0) { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not processed mbufs beyond good ones */ + if (k != n && k != 0) + mbuf_bad_move(mb, dr, n, n - k); + + inline_outb_mbuf_prepare(ss, mb, k); + return k; +} + +/* + * outbound for RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: + * actual processing is done by HW/PMD, just set flags and metadata. + */ +uint16_t +inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + inline_outb_mbuf_prepare(ss, mb, num); + return num; +} diff --git a/lib/librte_ipsec/ipsec_sqn.h b/lib/librte_ipsec/ipsec_sqn.h index a3ae7e2de..4ba079d75 100644 --- a/lib/librte_ipsec/ipsec_sqn.h +++ b/lib/librte_ipsec/ipsec_sqn.h @@ -56,21 +56,6 @@ sqn_low16(rte_be64_t sqn) #endif } -/* - * for given size, calculate required number of buckets. - */ -static uint32_t -replay_num_bucket(uint32_t wsz) -{ - uint32_t nb; - - nb = rte_align32pow2(RTE_ALIGN_MUL_CEIL(wsz, WINDOW_BUCKET_SIZE) / - WINDOW_BUCKET_SIZE); - nb = RTE_MAX(nb, (uint32_t)WINDOW_BUCKET_MIN); - - return nb; -} - /* * According to RFC4303 A2.1, determine the high-order bit of sequence number. * use 32bit arithmetic inside, return uint64_t. @@ -222,21 +207,6 @@ esn_inb_update_sqn(struct replay_sqn *rsn, const struct rte_ipsec_sa *sa, * between writer and readers. */ -/** - * Based on number of buckets calculated required size for the - * structure that holds replay window and sequence number (RSN) information. - */ -static size_t -rsn_size(uint32_t nb_bucket) -{ - size_t sz; - struct replay_sqn *rsn; - - sz = sizeof(*rsn) + nb_bucket * sizeof(rsn->window[0]); - sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); - return sz; -} - /** * Copy replay window and SQN. */ diff --git a/lib/librte_ipsec/meson.build b/lib/librte_ipsec/meson.build index d2427b809..18fb2a143 100644 --- a/lib/librte_ipsec/meson.build +++ b/lib/librte_ipsec/meson.build @@ -3,7 +3,7 @@ allow_experimental_apis = true -sources=files('sa.c', 'ses.c') +sources=files('esp_inb.c', 'esp_outb.c', 'sa.c', 'ses.c') install_headers = files('rte_ipsec.h', 'rte_ipsec_group.h', 'rte_ipsec_sa.h') diff --git a/lib/librte_ipsec/misc.h b/lib/librte_ipsec/misc.h new file mode 100644 index 000000000..8988b4903 --- /dev/null +++ b/lib/librte_ipsec/misc.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _MISC_H_ +#define _MISC_H_ + +/** + * @file misc.h + * Contains miscelaneous functions/structures/macros used internally + * by ipsec library. + */ + +/* + * Move bad (unprocessed) mbufs beyond the good (processed) ones. + * dr[] contains the indexes of bad mbufs insinde the mb[]. + */ +static inline void +mbuf_bad_move(struct rte_mbuf *mb[], const uint32_t dr[], uint32_t num, + uint32_t drn) +{ + uint32_t i, j, k; + struct rte_mbuf *drb[drn]; + + j = 0; + k = 0; + + /* copy bad ones into a temp place */ + for (i = 0; i != num; i++) { + if (j != drn && i == dr[j]) + drb[j++] = mb[i]; + else + mb[k++] = mb[i]; + } + + /* copy bad ones after the good ones */ + for (i = 0; i != drn; i++) + mb[k + i] = drb[i]; +} + +#endif /* _MISC_H_ */ diff --git a/lib/librte_ipsec/sa.c b/lib/librte_ipsec/sa.c index e4e3d7fc3..be6109768 100644 --- a/lib/librte_ipsec/sa.c +++ b/lib/librte_ipsec/sa.c @@ -12,6 +12,7 @@ #include "ipsec_sqn.h" #include "crypto.h" #include "iph.h" +#include "misc.h" #include "pad.h" #define MBUF_MAX_L2_LEN RTE_LEN2MASK(RTE_MBUF_L2_LEN_BITS, uint64_t) @@ -82,6 +83,36 @@ rte_ipsec_sa_type(const struct rte_ipsec_sa *sa) return sa->type; } +/** + * Based on number of buckets calculated required size for the + * structure that holds replay window and sequence number (RSN) information. + */ +static size_t +rsn_size(uint32_t nb_bucket) +{ + size_t sz; + struct replay_sqn *rsn; + + sz = sizeof(*rsn) + nb_bucket * sizeof(rsn->window[0]); + sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE); + return sz; +} + +/* + * for given size, calculate required number of buckets. + */ +static uint32_t +replay_num_bucket(uint32_t wsz) +{ + uint32_t nb; + + nb = rte_align32pow2(RTE_ALIGN_MUL_CEIL(wsz, WINDOW_BUCKET_SIZE) / + WINDOW_BUCKET_SIZE); + nb = RTE_MAX(nb, (uint32_t)WINDOW_BUCKET_MIN); + + return nb; +} + static int32_t ipsec_sa_size(uint64_t type, uint32_t *wnd_sz, uint32_t *nb_bucket) { @@ -411,569 +442,6 @@ rte_ipsec_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm, return sz; } -/* - * Move bad (unprocessed) mbufs beyond the good (processed) ones. - * dr[] contains the indexes of bad mbufs insinde the mb[]. - */ -static void -mbuf_bad_move(struct rte_mbuf *mb[], const uint32_t dr[], uint32_t num, - uint32_t drn) -{ - uint32_t i, j, k; - struct rte_mbuf *drb[drn]; - - j = 0; - k = 0; - - /* copy bad ones into a temp place */ - for (i = 0; i != num; i++) { - if (j != drn && i == dr[j]) - drb[j++] = mb[i]; - else - mb[k++] = mb[i]; - } - - /* copy bad ones after the good ones */ - for (i = 0; i != drn; i++) - mb[k + i] = drb[i]; -} - -/* - * setup crypto ops for LOOKASIDE_NONE (pure crypto) type of devices. - */ -static inline void -lksd_none_cop_prepare(struct rte_crypto_op *cop, - struct rte_cryptodev_sym_session *cs, struct rte_mbuf *mb) -{ - struct rte_crypto_sym_op *sop; - - sop = cop->sym; - cop->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; - cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - cop->sess_type = RTE_CRYPTO_OP_WITH_SESSION; - sop->m_src = mb; - __rte_crypto_sym_op_attach_sym_session(sop, cs); -} - -/* - * setup crypto op and crypto sym op for ESP outbound packet. - */ -static inline void -esp_outb_cop_prepare(struct rte_crypto_op *cop, - const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD], - const union sym_op_data *icv, uint32_t hlen, uint32_t plen) -{ - struct rte_crypto_sym_op *sop; - struct aead_gcm_iv *gcm; - - /* fill sym op fields */ - sop = cop->sym; - - /* AEAD (AES_GCM) case */ - if (sa->aad_len != 0) { - sop->aead.data.offset = sa->ctp.cipher.offset + hlen; - sop->aead.data.length = sa->ctp.cipher.length + plen; - sop->aead.digest.data = icv->va; - sop->aead.digest.phys_addr = icv->pa; - sop->aead.aad.data = icv->va + sa->icv_len; - sop->aead.aad.phys_addr = icv->pa + sa->icv_len; - - /* fill AAD IV (located inside crypto op) */ - gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *, - sa->iv_ofs); - aead_gcm_iv_fill(gcm, ivp[0], sa->salt); - /* CRYPT+AUTH case */ - } else { - sop->cipher.data.offset = sa->ctp.cipher.offset + hlen; - sop->cipher.data.length = sa->ctp.cipher.length + plen; - sop->auth.data.offset = sa->ctp.auth.offset + hlen; - sop->auth.data.length = sa->ctp.auth.length + plen; - sop->auth.digest.data = icv->va; - sop->auth.digest.phys_addr = icv->pa; - } -} - -/* - * setup/update packet data and metadata for ESP outbound tunnel case. - */ -static inline int32_t -esp_outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, - const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, - union sym_op_data *icv) -{ - uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen; - struct rte_mbuf *ml; - struct esp_hdr *esph; - struct esp_tail *espt; - char *ph, *pt; - uint64_t *iv; - - /* calculate extra header space required */ - hlen = sa->hdr_len + sa->iv_len + sizeof(*esph); - - /* size of ipsec protected data */ - l2len = mb->l2_len; - plen = mb->pkt_len - l2len; - - /* number of bytes to encrypt */ - clen = plen + sizeof(*espt); - clen = RTE_ALIGN_CEIL(clen, sa->pad_align); - - /* pad length + esp tail */ - pdlen = clen - plen; - tlen = pdlen + sa->icv_len; - - /* do append and prepend */ - ml = rte_pktmbuf_lastseg(mb); - if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml)) - return -ENOSPC; - - /* prepend header */ - ph = rte_pktmbuf_prepend(mb, hlen - l2len); - if (ph == NULL) - return -ENOSPC; - - /* append tail */ - pdofs = ml->data_len; - ml->data_len += tlen; - mb->pkt_len += tlen; - pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs); - - /* update pkt l2/l3 len */ - mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) | - sa->tx_offload.val; - - /* copy tunnel pkt header */ - rte_memcpy(ph, sa->hdr, sa->hdr_len); - - /* update original and new ip header fields */ - update_tun_l3hdr(sa, ph + sa->hdr_l3_off, mb->pkt_len, sa->hdr_l3_off, - sqn_low16(sqc)); - - /* update spi, seqn and iv */ - esph = (struct esp_hdr *)(ph + sa->hdr_len); - iv = (uint64_t *)(esph + 1); - copy_iv(iv, ivp, sa->iv_len); - - esph->spi = sa->spi; - esph->seq = sqn_low32(sqc); - - /* offset for ICV */ - pdofs += pdlen + sa->sqh_len; - - /* pad length */ - pdlen -= sizeof(*espt); - - /* copy padding data */ - rte_memcpy(pt, esp_pad_bytes, pdlen); - - /* update esp trailer */ - espt = (struct esp_tail *)(pt + pdlen); - espt->pad_len = pdlen; - espt->next_proto = sa->proto; - - icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); - icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); - - return clen; -} - -/* - * for pure cryptodev (lookaside none) depending on SA settings, - * we might have to write some extra data to the packet. - */ -static inline void -outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, - const union sym_op_data *icv) -{ - uint32_t *psqh; - struct aead_gcm_aad *aad; - - /* insert SQN.hi between ESP trailer and ICV */ - if (sa->sqh_len != 0) { - psqh = (uint32_t *)(icv->va - sa->sqh_len); - psqh[0] = sqn_hi32(sqc); - } - - /* - * fill IV and AAD fields, if any (aad fields are placed after icv), - * right now we support only one AEAD algorithm: AES-GCM . - */ - if (sa->aad_len != 0) { - aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); - aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); - } -} - -/* - * setup/update packets and crypto ops for ESP outbound tunnel case. - */ -static uint16_t -outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - struct rte_crypto_op *cop[], uint16_t num) -{ - int32_t rc; - uint32_t i, k, n; - uint64_t sqn; - rte_be64_t sqc; - struct rte_ipsec_sa *sa; - struct rte_cryptodev_sym_session *cs; - union sym_op_data icv; - uint64_t iv[IPSEC_MAX_IV_QWORD]; - uint32_t dr[num]; - - sa = ss->sa; - cs = ss->crypto.ses; - - n = num; - sqn = esn_outb_update_sqn(sa, &n); - if (n != num) - rte_errno = EOVERFLOW; - - k = 0; - for (i = 0; i != n; i++) { - - sqc = rte_cpu_to_be_64(sqn + i); - gen_iv(iv, sqc); - - /* try to update the packet itself */ - rc = esp_outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv); - - /* success, setup crypto op */ - if (rc >= 0) { - outb_pkt_xprepare(sa, sqc, &icv); - lksd_none_cop_prepare(cop[k], cs, mb[i]); - esp_outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc); - k++; - /* failure, put packet into the death-row */ - } else { - dr[i - k] = i; - rte_errno = -rc; - } - } - - /* copy not prepared mbufs beyond good ones */ - if (k != n && k != 0) - mbuf_bad_move(mb, dr, n, n - k); - - return k; -} - -/* - * setup/update packet data and metadata for ESP outbound transport case. - */ -static inline int32_t -esp_outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, - const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, - uint32_t l2len, uint32_t l3len, union sym_op_data *icv) -{ - uint8_t np; - uint32_t clen, hlen, pdlen, pdofs, plen, tlen, uhlen; - struct rte_mbuf *ml; - struct esp_hdr *esph; - struct esp_tail *espt; - char *ph, *pt; - uint64_t *iv; - - uhlen = l2len + l3len; - plen = mb->pkt_len - uhlen; - - /* calculate extra header space required */ - hlen = sa->iv_len + sizeof(*esph); - - /* number of bytes to encrypt */ - clen = plen + sizeof(*espt); - clen = RTE_ALIGN_CEIL(clen, sa->pad_align); - - /* pad length + esp tail */ - pdlen = clen - plen; - tlen = pdlen + sa->icv_len; - - /* do append and insert */ - ml = rte_pktmbuf_lastseg(mb); - if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml)) - return -ENOSPC; - - /* prepend space for ESP header */ - ph = rte_pktmbuf_prepend(mb, hlen); - if (ph == NULL) - return -ENOSPC; - - /* append tail */ - pdofs = ml->data_len; - ml->data_len += tlen; - mb->pkt_len += tlen; - pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs); - - /* shift L2/L3 headers */ - insert_esph(ph, ph + hlen, uhlen); - - /* update ip header fields */ - np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len, l2len, l3len, - IPPROTO_ESP); - - /* update spi, seqn and iv */ - esph = (struct esp_hdr *)(ph + uhlen); - iv = (uint64_t *)(esph + 1); - copy_iv(iv, ivp, sa->iv_len); - - esph->spi = sa->spi; - esph->seq = sqn_low32(sqc); - - /* offset for ICV */ - pdofs += pdlen + sa->sqh_len; - - /* pad length */ - pdlen -= sizeof(*espt); - - /* copy padding data */ - rte_memcpy(pt, esp_pad_bytes, pdlen); - - /* update esp trailer */ - espt = (struct esp_tail *)(pt + pdlen); - espt->pad_len = pdlen; - espt->next_proto = np; - - icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); - icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); - - return clen; -} - -/* - * setup/update packets and crypto ops for ESP outbound transport case. - */ -static uint16_t -outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - struct rte_crypto_op *cop[], uint16_t num) -{ - int32_t rc; - uint32_t i, k, n, l2, l3; - uint64_t sqn; - rte_be64_t sqc; - struct rte_ipsec_sa *sa; - struct rte_cryptodev_sym_session *cs; - union sym_op_data icv; - uint64_t iv[IPSEC_MAX_IV_QWORD]; - uint32_t dr[num]; - - sa = ss->sa; - cs = ss->crypto.ses; - - n = num; - sqn = esn_outb_update_sqn(sa, &n); - if (n != num) - rte_errno = EOVERFLOW; - - k = 0; - for (i = 0; i != n; i++) { - - l2 = mb[i]->l2_len; - l3 = mb[i]->l3_len; - - sqc = rte_cpu_to_be_64(sqn + i); - gen_iv(iv, sqc); - - /* try to update the packet itself */ - rc = esp_outb_trs_pkt_prepare(sa, sqc, iv, mb[i], - l2, l3, &icv); - - /* success, setup crypto op */ - if (rc >= 0) { - outb_pkt_xprepare(sa, sqc, &icv); - lksd_none_cop_prepare(cop[k], cs, mb[i]); - esp_outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc); - k++; - /* failure, put packet into the death-row */ - } else { - dr[i - k] = i; - rte_errno = -rc; - } - } - - /* copy not prepared mbufs beyond good ones */ - if (k != n && k != 0) - mbuf_bad_move(mb, dr, n, n - k); - - return k; -} - -/* - * setup crypto op and crypto sym op for ESP inbound tunnel packet. - */ -static inline int32_t -esp_inb_tun_cop_prepare(struct rte_crypto_op *cop, - const struct rte_ipsec_sa *sa, struct rte_mbuf *mb, - const union sym_op_data *icv, uint32_t pofs, uint32_t plen) -{ - struct rte_crypto_sym_op *sop; - struct aead_gcm_iv *gcm; - uint64_t *ivc, *ivp; - uint32_t clen; - - clen = plen - sa->ctp.cipher.length; - if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0) - return -EINVAL; - - /* fill sym op fields */ - sop = cop->sym; - - /* AEAD (AES_GCM) case */ - if (sa->aad_len != 0) { - sop->aead.data.offset = pofs + sa->ctp.cipher.offset; - sop->aead.data.length = clen; - sop->aead.digest.data = icv->va; - sop->aead.digest.phys_addr = icv->pa; - sop->aead.aad.data = icv->va + sa->icv_len; - sop->aead.aad.phys_addr = icv->pa + sa->icv_len; - - /* fill AAD IV (located inside crypto op) */ - gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *, - sa->iv_ofs); - ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *, - pofs + sizeof(struct esp_hdr)); - aead_gcm_iv_fill(gcm, ivp[0], sa->salt); - /* CRYPT+AUTH case */ - } else { - sop->cipher.data.offset = pofs + sa->ctp.cipher.offset; - sop->cipher.data.length = clen; - sop->auth.data.offset = pofs + sa->ctp.auth.offset; - sop->auth.data.length = plen - sa->ctp.auth.length; - sop->auth.digest.data = icv->va; - sop->auth.digest.phys_addr = icv->pa; - - /* copy iv from the input packet to the cop */ - ivc = rte_crypto_op_ctod_offset(cop, uint64_t *, sa->iv_ofs); - ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *, - pofs + sizeof(struct esp_hdr)); - copy_iv(ivc, ivp, sa->iv_len); - } - return 0; -} - -/* - * for pure cryptodev (lookaside none) depending on SA settings, - * we might have to write some extra data to the packet. - */ -static inline void -inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, - const union sym_op_data *icv) -{ - struct aead_gcm_aad *aad; - - /* insert SQN.hi between ESP trailer and ICV */ - if (sa->sqh_len != 0) - insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len); - - /* - * fill AAD fields, if any (aad fields are placed after icv), - * right now we support only one AEAD algorithm: AES-GCM. - */ - if (sa->aad_len != 0) { - aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); - aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); - } -} - -/* - * setup/update packet data and metadata for ESP inbound tunnel case. - */ -static inline int32_t -esp_inb_tun_pkt_prepare(const struct rte_ipsec_sa *sa, - const struct replay_sqn *rsn, struct rte_mbuf *mb, - uint32_t hlen, union sym_op_data *icv) -{ - int32_t rc; - uint64_t sqn; - uint32_t icv_ofs, plen; - struct rte_mbuf *ml; - struct esp_hdr *esph; - - esph = rte_pktmbuf_mtod_offset(mb, struct esp_hdr *, hlen); - - /* - * retrieve and reconstruct SQN, then check it, then - * convert it back into network byte order. - */ - sqn = rte_be_to_cpu_32(esph->seq); - if (IS_ESN(sa)) - sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz); - - rc = esn_inb_check_sqn(rsn, sa, sqn); - if (rc != 0) - return rc; - - sqn = rte_cpu_to_be_64(sqn); - - /* start packet manipulation */ - plen = mb->pkt_len; - plen = plen - hlen; - - ml = rte_pktmbuf_lastseg(mb); - icv_ofs = ml->data_len - sa->icv_len + sa->sqh_len; - - /* we have to allocate space for AAD somewhere, - * right now - just use free trailing space at the last segment. - * Would probably be more convenient to reserve space for AAD - * inside rte_crypto_op itself - * (again for IV space is already reserved inside cop). - */ - if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml)) - return -ENOSPC; - - icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs); - icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs); - - inb_pkt_xprepare(sa, sqn, icv); - return plen; -} - -/* - * setup/update packets and crypto ops for ESP inbound case. - */ -static uint16_t -inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - struct rte_crypto_op *cop[], uint16_t num) -{ - int32_t rc; - uint32_t i, k, hl; - struct rte_ipsec_sa *sa; - struct rte_cryptodev_sym_session *cs; - struct replay_sqn *rsn; - union sym_op_data icv; - uint32_t dr[num]; - - sa = ss->sa; - cs = ss->crypto.ses; - rsn = rsn_acquire(sa); - - k = 0; - for (i = 0; i != num; i++) { - - hl = mb[i]->l2_len + mb[i]->l3_len; - rc = esp_inb_tun_pkt_prepare(sa, rsn, mb[i], hl, &icv); - if (rc >= 0) { - lksd_none_cop_prepare(cop[k], cs, mb[i]); - rc = esp_inb_tun_cop_prepare(cop[k], sa, mb[i], &icv, - hl, rc); - } - - k += (rc == 0); - if (rc != 0) { - dr[i - k] = i; - rte_errno = -rc; - } - } - - rsn_release(sa, rsn); - - /* copy not prepared mbufs beyond good ones */ - if (k != num && k != 0) - mbuf_bad_move(mb, dr, num, num - k); - - return k; -} - /* * setup crypto ops for LOOKASIDE_PROTO type of devices. */ @@ -1008,265 +476,6 @@ lksd_proto_prepare(const struct rte_ipsec_session *ss, return num; } -/* - * process ESP inbound tunnel packet. - */ -static inline int -esp_inb_tun_single_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb, - uint32_t *sqn) -{ - uint32_t hlen, icv_len, tlen; - struct esp_hdr *esph; - struct esp_tail *espt; - struct rte_mbuf *ml; - char *pd; - - if (mb->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) - return -EBADMSG; - - icv_len = sa->icv_len; - - ml = rte_pktmbuf_lastseg(mb); - espt = rte_pktmbuf_mtod_offset(ml, struct esp_tail *, - ml->data_len - icv_len - sizeof(*espt)); - - /* - * check padding and next proto. - * return an error if something is wrong. - */ - pd = (char *)espt - espt->pad_len; - if (espt->next_proto != sa->proto || - memcmp(pd, esp_pad_bytes, espt->pad_len)) - return -EINVAL; - - /* cut of ICV, ESP tail and padding bytes */ - tlen = icv_len + sizeof(*espt) + espt->pad_len; - ml->data_len -= tlen; - mb->pkt_len -= tlen; - - /* cut of L2/L3 headers, ESP header and IV */ - hlen = mb->l2_len + mb->l3_len; - esph = rte_pktmbuf_mtod_offset(mb, struct esp_hdr *, hlen); - rte_pktmbuf_adj(mb, hlen + sa->ctp.cipher.offset); - - /* retrieve SQN for later check */ - *sqn = rte_be_to_cpu_32(esph->seq); - - /* reset mbuf metatdata: L2/L3 len, packet type */ - mb->packet_type = RTE_PTYPE_UNKNOWN; - mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) | - sa->tx_offload.val; - - /* clear the PKT_RX_SEC_OFFLOAD flag if set */ - mb->ol_flags &= ~(mb->ol_flags & PKT_RX_SEC_OFFLOAD); - return 0; -} - -/* - * process ESP inbound transport packet. - */ -static inline int -esp_inb_trs_single_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb, - uint32_t *sqn) -{ - uint32_t hlen, icv_len, l2len, l3len, tlen; - struct esp_hdr *esph; - struct esp_tail *espt; - struct rte_mbuf *ml; - char *np, *op, *pd; - - if (mb->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) - return -EBADMSG; - - icv_len = sa->icv_len; - - ml = rte_pktmbuf_lastseg(mb); - espt = rte_pktmbuf_mtod_offset(ml, struct esp_tail *, - ml->data_len - icv_len - sizeof(*espt)); - - /* check padding, return an error if something is wrong. */ - pd = (char *)espt - espt->pad_len; - if (memcmp(pd, esp_pad_bytes, espt->pad_len)) - return -EINVAL; - - /* cut of ICV, ESP tail and padding bytes */ - tlen = icv_len + sizeof(*espt) + espt->pad_len; - ml->data_len -= tlen; - mb->pkt_len -= tlen; - - /* retrieve SQN for later check */ - l2len = mb->l2_len; - l3len = mb->l3_len; - hlen = l2len + l3len; - op = rte_pktmbuf_mtod(mb, char *); - esph = (struct esp_hdr *)(op + hlen); - *sqn = rte_be_to_cpu_32(esph->seq); - - /* cut off ESP header and IV, update L3 header */ - np = rte_pktmbuf_adj(mb, sa->ctp.cipher.offset); - remove_esph(np, op, hlen); - update_trs_l3hdr(sa, np + l2len, mb->pkt_len, l2len, l3len, - espt->next_proto); - - /* reset mbuf packet type */ - mb->packet_type &= (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK); - - /* clear the PKT_RX_SEC_OFFLOAD flag if set */ - mb->ol_flags &= ~(mb->ol_flags & PKT_RX_SEC_OFFLOAD); - return 0; -} - -/* - * for group of ESP inbound packets perform SQN check and update. - */ -static inline uint16_t -esp_inb_rsn_update(struct rte_ipsec_sa *sa, const uint32_t sqn[], - uint32_t dr[], uint16_t num) -{ - uint32_t i, k; - struct replay_sqn *rsn; - - rsn = rsn_update_start(sa); - - k = 0; - for (i = 0; i != num; i++) { - if (esn_inb_update_sqn(rsn, sa, sqn[i]) == 0) - k++; - else - dr[i - k] = i; - } - - rsn_update_finish(sa, rsn); - return k; -} - -/* - * process group of ESP inbound tunnel packets. - */ -static uint16_t -inb_tun_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - uint16_t num) -{ - uint32_t i, k, n; - struct rte_ipsec_sa *sa; - uint32_t sqn[num]; - uint32_t dr[num]; - - sa = ss->sa; - - /* process packets, extract seq numbers */ - - k = 0; - for (i = 0; i != num; i++) { - /* good packet */ - if (esp_inb_tun_single_pkt_process(sa, mb[i], sqn + k) == 0) - k++; - /* bad packet, will drop from furhter processing */ - else - dr[i - k] = i; - } - - /* handle unprocessed mbufs */ - if (k != num && k != 0) - mbuf_bad_move(mb, dr, num, num - k); - - /* update SQN and replay winow */ - n = esp_inb_rsn_update(sa, sqn, dr, k); - - /* handle mbufs with wrong SQN */ - if (n != k && n != 0) - mbuf_bad_move(mb, dr, k, k - n); - - if (n != num) - rte_errno = EBADMSG; - - return n; -} - -/* - * process group of ESP inbound transport packets. - */ -static uint16_t -inb_trs_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - uint16_t num) -{ - uint32_t i, k, n; - uint32_t sqn[num]; - struct rte_ipsec_sa *sa; - uint32_t dr[num]; - - sa = ss->sa; - - /* process packets, extract seq numbers */ - - k = 0; - for (i = 0; i != num; i++) { - /* good packet */ - if (esp_inb_trs_single_pkt_process(sa, mb[i], sqn + k) == 0) - k++; - /* bad packet, will drop from furhter processing */ - else - dr[i - k] = i; - } - - /* handle unprocessed mbufs */ - if (k != num && k != 0) - mbuf_bad_move(mb, dr, num, num - k); - - /* update SQN and replay winow */ - n = esp_inb_rsn_update(sa, sqn, dr, k); - - /* handle mbufs with wrong SQN */ - if (n != k && n != 0) - mbuf_bad_move(mb, dr, k, k - n); - - if (n != num) - rte_errno = EBADMSG; - - return n; -} - -/* - * process outbound packets for SA with ESN support, - * for algorithms that require SQN.hibits to be implictly included - * into digest computation. - * In that case we have to move ICV bytes back to their proper place. - */ -static uint16_t -outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - uint16_t num) -{ - uint32_t i, k, icv_len, *icv; - struct rte_mbuf *ml; - struct rte_ipsec_sa *sa; - uint32_t dr[num]; - - sa = ss->sa; - - k = 0; - icv_len = sa->icv_len; - - for (i = 0; i != num; i++) { - if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) { - ml = rte_pktmbuf_lastseg(mb[i]); - icv = rte_pktmbuf_mtod_offset(ml, void *, - ml->data_len - icv_len); - remove_sqh(icv, icv_len); - k++; - } else - dr[i - k] = i; - } - - /* handle unprocessed mbufs */ - if (k != num) { - rte_errno = EBADMSG; - if (k != 0) - mbuf_bad_move(mb, dr, num, num - k); - } - - return k; -} - /* * simplest pkt process routine: * all actual processing is already done by HW/PMD, @@ -1303,142 +512,6 @@ pkt_flag_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], return k; } -/* - * prepare packets for inline ipsec processing: - * set ol_flags and attach metadata. - */ -static inline void -inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss, - struct rte_mbuf *mb[], uint16_t num) -{ - uint32_t i, ol_flags; - - ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA; - for (i = 0; i != num; i++) { - - mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD; - if (ol_flags != 0) - rte_security_set_pkt_metadata(ss->security.ctx, - ss->security.ses, mb[i], NULL); - } -} - -/* - * process group of ESP outbound tunnel packets destined for - * INLINE_CRYPTO type of device. - */ -static uint16_t -inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss, - struct rte_mbuf *mb[], uint16_t num) -{ - int32_t rc; - uint32_t i, k, n; - uint64_t sqn; - rte_be64_t sqc; - struct rte_ipsec_sa *sa; - union sym_op_data icv; - uint64_t iv[IPSEC_MAX_IV_QWORD]; - uint32_t dr[num]; - - sa = ss->sa; - - n = num; - sqn = esn_outb_update_sqn(sa, &n); - if (n != num) - rte_errno = EOVERFLOW; - - k = 0; - for (i = 0; i != n; i++) { - - sqc = rte_cpu_to_be_64(sqn + i); - gen_iv(iv, sqc); - - /* try to update the packet itself */ - rc = esp_outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv); - - k += (rc >= 0); - - /* failure, put packet into the death-row */ - if (rc < 0) { - dr[i - k] = i; - rte_errno = -rc; - } - } - - /* copy not processed mbufs beyond good ones */ - if (k != n && k != 0) - mbuf_bad_move(mb, dr, n, n - k); - - inline_outb_mbuf_prepare(ss, mb, k); - return k; -} - -/* - * process group of ESP outbound transport packets destined for - * INLINE_CRYPTO type of device. - */ -static uint16_t -inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss, - struct rte_mbuf *mb[], uint16_t num) -{ - int32_t rc; - uint32_t i, k, n, l2, l3; - uint64_t sqn; - rte_be64_t sqc; - struct rte_ipsec_sa *sa; - union sym_op_data icv; - uint64_t iv[IPSEC_MAX_IV_QWORD]; - uint32_t dr[num]; - - sa = ss->sa; - - n = num; - sqn = esn_outb_update_sqn(sa, &n); - if (n != num) - rte_errno = EOVERFLOW; - - k = 0; - for (i = 0; i != n; i++) { - - l2 = mb[i]->l2_len; - l3 = mb[i]->l3_len; - - sqc = rte_cpu_to_be_64(sqn + i); - gen_iv(iv, sqc); - - /* try to update the packet itself */ - rc = esp_outb_trs_pkt_prepare(sa, sqc, iv, mb[i], - l2, l3, &icv); - - k += (rc >= 0); - - /* failure, put packet into the death-row */ - if (rc < 0) { - dr[i - k] = i; - rte_errno = -rc; - } - } - - /* copy not processed mbufs beyond good ones */ - if (k != n && k != 0) - mbuf_bad_move(mb, dr, n, n - k); - - inline_outb_mbuf_prepare(ss, mb, k); - return k; -} - -/* - * outbound for RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: - * actual processing is done by HW/PMD, just set flags and metadata. - */ -static uint16_t -outb_inline_proto_process(const struct rte_ipsec_session *ss, - struct rte_mbuf *mb[], uint16_t num) -{ - inline_outb_mbuf_prepare(ss, mb, num); - return num; -} - /* * Select packet processing function for session on LOOKASIDE_NONE * type of device. @@ -1456,23 +529,23 @@ lksd_none_pkt_func_select(const struct rte_ipsec_sa *sa, switch (sa->type & msk) { case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4): case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6): - pf->prepare = inb_pkt_prepare; - pf->process = inb_tun_pkt_process; + pf->prepare = esp_inb_pkt_prepare; + pf->process = esp_inb_tun_pkt_process; break; case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS): - pf->prepare = inb_pkt_prepare; - pf->process = inb_trs_pkt_process; + pf->prepare = esp_inb_pkt_prepare; + pf->process = esp_inb_trs_pkt_process; break; case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4): case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6): - pf->prepare = outb_tun_prepare; + pf->prepare = esp_outb_tun_prepare; pf->process = (sa->sqh_len != 0) ? - outb_sqh_process : pkt_flag_process; + esp_outb_sqh_process : pkt_flag_process; break; case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS): - pf->prepare = outb_trs_prepare; + pf->prepare = esp_outb_trs_prepare; pf->process = (sa->sqh_len != 0) ? - outb_sqh_process : pkt_flag_process; + esp_outb_sqh_process : pkt_flag_process; break; default: rc = -ENOTSUP; @@ -1498,10 +571,10 @@ inline_crypto_pkt_func_select(const struct rte_ipsec_sa *sa, switch (sa->type & msk) { case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4): case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6): - pf->process = inb_tun_pkt_process; + pf->process = esp_inb_tun_pkt_process; break; case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS): - pf->process = inb_trs_pkt_process; + pf->process = esp_inb_trs_pkt_process; break; case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4): case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6): @@ -1542,7 +615,7 @@ ipsec_sa_pkt_func_select(const struct rte_ipsec_session *ss, RTE_IPSEC_SATP_DIR_IB) pf->process = pkt_flag_process; else - pf->process = outb_inline_proto_process; + pf->process = inline_proto_outb_pkt_process; break; case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: pf->prepare = lksd_proto_prepare; diff --git a/lib/librte_ipsec/sa.h b/lib/librte_ipsec/sa.h index 133a35d83..93ff7bdf1 100644 --- a/lib/librte_ipsec/sa.h +++ b/lib/librte_ipsec/sa.h @@ -108,4 +108,44 @@ int ipsec_sa_pkt_func_select(const struct rte_ipsec_session *ss, const struct rte_ipsec_sa *sa, struct rte_ipsec_sa_pkt_func *pf); +/* inbound processing */ + +uint16_t +esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num); + +uint16_t +esp_inb_tun_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +esp_inb_trs_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +/* outbound processing */ + +uint16_t +esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num); + +uint16_t +esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + struct rte_crypto_op *cop[], uint16_t num); + +uint16_t +esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], + uint16_t num); + +uint16_t +inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + #endif /* _SA_H_ */ -- 2.17.1