From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 40A73A2EEB for ; Mon, 7 Oct 2019 18:30:25 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id DE73B1D381; Mon, 7 Oct 2019 18:29:18 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id 93FEA1D15A for ; Mon, 7 Oct 2019 18:29:10 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 07 Oct 2019 09:29:10 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.67,268,1566889200"; d="scan'208";a="393082053" Received: from silpixa00398673.ir.intel.com (HELO silpixa00398673.ger.corp.intel.com) ([10.237.223.136]) by fmsmga005.fm.intel.com with ESMTP; 07 Oct 2019 09:29:08 -0700 From: Fan Zhang To: dev@dpdk.org Cc: konstantin.ananyev@intel.com, declan.doherty@intel.com, akhil.goyal@nxp.com, Fan Zhang Date: Mon, 7 Oct 2019 17:28:48 +0100 Message-Id: <20191007162850.60552-9-roy.fan.zhang@intel.com> X-Mailer: git-send-email 2.14.5 In-Reply-To: <20191007162850.60552-1-roy.fan.zhang@intel.com> References: <20190906131330.40185-1-roy.fan.zhang@intel.com> <20191007162850.60552-1-roy.fan.zhang@intel.com> Subject: [dpdk-dev] [PATCH v2 08/10] ipsec: add rte_security cpu_crypto action support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch updates the ipsec library to handle the newly introduced RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO action. Signed-off-by: Fan Zhang --- lib/librte_ipsec/crypto.h | 24 +++ lib/librte_ipsec/esp_inb.c | 200 ++++++++++++++++++++++-- lib/librte_ipsec/esp_outb.c | 369 +++++++++++++++++++++++++++++++++++++++++--- lib/librte_ipsec/sa.c | 53 ++++++- lib/librte_ipsec/sa.h | 29 ++++ lib/librte_ipsec/ses.c | 4 +- 6 files changed, 643 insertions(+), 36 deletions(-) diff --git a/lib/librte_ipsec/crypto.h b/lib/librte_ipsec/crypto.h index f8fbf8d4f..901c8c7de 100644 --- a/lib/librte_ipsec/crypto.h +++ b/lib/librte_ipsec/crypto.h @@ -179,4 +179,28 @@ lksd_none_cop_prepare(struct rte_crypto_op *cop, __rte_crypto_sym_op_attach_sym_session(sop, cs); } +typedef void* (*_set_icv_f)(void *val, struct rte_mbuf *ml, uint32_t icv_off); + +static inline void * +set_icv_va_pa(void *val, struct rte_mbuf *ml, uint32_t icv_off) +{ + union sym_op_data *icv = val; + + icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_off); + icv->pa = rte_pktmbuf_iova_offset(ml, icv_off); + + return icv->va; +} + +static inline void * +set_icv_va(__rte_unused void *val, __rte_unused struct rte_mbuf *ml, + __rte_unused uint32_t icv_off) +{ + void **icv_va = val; + + *icv_va = rte_pktmbuf_mtod_offset(ml, void *, icv_off); + + return *icv_va; +} + #endif /* _CRYPTO_H_ */ diff --git a/lib/librte_ipsec/esp_inb.c b/lib/librte_ipsec/esp_inb.c index 8e3ecbc64..c4476e819 100644 --- a/lib/librte_ipsec/esp_inb.c +++ b/lib/librte_ipsec/esp_inb.c @@ -105,6 +105,78 @@ inb_cop_prepare(struct rte_crypto_op *cop, } } +static inline int +inb_cpu_crypto_proc_prepare(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb, + uint32_t pofs, uint32_t plen, + struct rte_security_vec *buf, struct iovec *cur_vec, + void *iv) +{ + struct rte_mbuf *ms; + struct iovec *vec = cur_vec; + struct aead_gcm_iv *gcm; + struct aesctr_cnt_blk *ctr; + uint64_t *ivp; + uint32_t algo; + uint32_t left; + uint32_t off = 0, n_seg = 0; + + ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *, + pofs + sizeof(struct rte_esp_hdr)); + algo = sa->algo_type; + + switch (algo) { + case ALGO_TYPE_AES_GCM: + gcm = (struct aead_gcm_iv *)iv; + aead_gcm_iv_fill(gcm, ivp[0], sa->salt); + off = sa->ctp.cipher.offset + pofs; + left = plen - sa->ctp.cipher.length; + break; + case ALGO_TYPE_AES_CBC: + case ALGO_TYPE_3DES_CBC: + copy_iv(iv, ivp, sa->iv_len); + off = sa->ctp.auth.offset + pofs; + left = plen - sa->ctp.auth.length; + break; + case ALGO_TYPE_AES_CTR: + copy_iv(iv, ivp, sa->iv_len); + off = sa->ctp.auth.offset + pofs; + left = plen - sa->ctp.auth.length; + ctr = (struct aesctr_cnt_blk *)iv; + aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt); + break; + case ALGO_TYPE_NULL: + left = plen - sa->ctp.cipher.length; + break; + default: + return -EINVAL; + } + + ms = mbuf_get_seg_ofs(mb, &off); + if (!ms) + return -1; + + while (n_seg < RTE_LIBRTE_IP_FRAG_MAX_FRAG && left && ms) { + uint32_t len = RTE_MIN(left, ms->data_len - off); + + vec->iov_base = rte_pktmbuf_mtod_offset(ms, void *, off); + vec->iov_len = len; + + left -= len; + vec++; + n_seg++; + ms = ms->next; + off = 0; + } + + if (left) + return -1; + + buf->vec = cur_vec; + buf->num = n_seg; + + return n_seg; +} + /* * Helper function for prepare() to deal with situation when * ICV is spread by two segments. Tries to move ICV completely into the @@ -139,20 +211,21 @@ move_icv(struct rte_mbuf *ml, uint32_t ofs) */ static inline void inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, - const union sym_op_data *icv) + uint8_t *icv_va, void *aad_buf, uint32_t aad_off) { struct aead_gcm_aad *aad; /* insert SQN.hi between ESP trailer and ICV */ if (sa->sqh_len != 0) - insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len); + insert_sqh(sqn_hi32(sqc), icv_va, sa->icv_len); /* * fill AAD fields, if any (aad fields are placed after icv), * right now we support only one AEAD algorithm: AES-GCM. */ if (sa->aad_len != 0) { - aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); + aad = aad_buf ? aad_buf : + (struct aead_gcm_aad *)(icv_va + aad_off); aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); } } @@ -162,13 +235,15 @@ inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, */ static inline int32_t inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn, - struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv) + struct rte_mbuf *mb, uint32_t hlen, _set_icv_f set_icv, void *icv_val, + void *aad_buf) { int32_t rc; uint64_t sqn; uint32_t clen, icv_len, icv_ofs, plen; struct rte_mbuf *ml; struct rte_esp_hdr *esph; + void *icv_va; esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen); @@ -226,8 +301,8 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn, if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml)) return -ENOSPC; - icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs); - icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs); + icv_va = set_icv(icv_val, ml, icv_ofs); + inb_pkt_xprepare(sa, sqn, icv_va, aad_buf, sa->icv_len); /* * if esn is used then high-order 32 bits are also used in ICV @@ -238,7 +313,6 @@ inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn, mb->pkt_len += sa->sqh_len; ml->data_len += sa->sqh_len; - inb_pkt_xprepare(sa, sqn, icv); return plen; } @@ -265,7 +339,8 @@ esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], for (i = 0; i != num; i++) { hl = mb[i]->l2_len + mb[i]->l3_len; - rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv); + rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va_pa, + (void *)&icv, NULL); if (rc >= 0) { lksd_none_cop_prepare(cop[k], cs, mb[i]); inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc); @@ -512,7 +587,6 @@ tun_process(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb[], return k; } - /* * *process* function for tunnel packets */ @@ -625,6 +699,114 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[], return n; } +/* + * process packets using sync crypto engine + */ +static uint16_t +esp_inb_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num, + esp_inb_process_t process) +{ + int32_t rc; + uint32_t i, hl, n, p; + struct rte_ipsec_sa *sa; + struct replay_sqn *rsn; + void *icv_va; + uint32_t sqn[num]; + uint32_t dr[num]; + uint8_t sqh_len; + + /* cpu crypto specific variables */ + struct rte_security_vec buf[num]; + struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num]; + uint32_t vec_idx = 0; + uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD]; + void *iv[num]; + int status[num]; + uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)]; + void *aad[num]; + void *digest[num]; + uint32_t k; + + sa = ss->sa; + rsn = rsn_acquire(sa); + sqh_len = sa->sqh_len; + + k = 0; + for (i = 0; i != num; i++) { + hl = mb[i]->l2_len + mb[i]->l3_len; + rc = inb_pkt_prepare(sa, rsn, mb[i], hl, set_icv_va, + (void *)&icv_va, (void *)aad_buf[k]); + if (rc >= 0) { + iv[k] = (void *)iv_buf[k]; + aad[k] = (void *)aad_buf[k]; + digest[k] = (void *)icv_va; + + rc = inb_cpu_crypto_proc_prepare(sa, mb[i], hl, + rc, &buf[k], &vec[vec_idx], iv[k]); + if (rc < 0) { + dr[i - k] = i; + continue; + } + + vec_idx += rc; + k++; + } else + dr[i - k] = i; + } + + /* copy not prepared mbufs beyond good ones */ + if (k != num) { + rte_errno = EBADMSG; + + if (unlikely(k == 0)) + return 0; + + move_bad_mbufs(mb, dr, num, num - k); + } + + /* process the packets */ + n = 0; + rc = rte_security_process_cpu_crypto_bulk(ss->security.ctx, + ss->security.ses, buf, iv, aad, digest, status, k); + /* move failed process packets to dr */ + for (i = 0; i < k; i++) { + if (status[i]) { + dr[n++] = i; + rte_errno = EBADMSG; + } + } + + /* move bad packets to the back */ + if (n) + move_bad_mbufs(mb, dr, k, n); + + /* process packets */ + p = process(sa, mb, sqn, dr, k - n, sqh_len); + + if (p != k - n && p != 0) + move_bad_mbufs(mb, dr, k - n, k - n - p); + + if (p != num) + rte_errno = EBADMSG; + + return p; +} + +uint16_t +esp_inb_tun_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + return esp_inb_cpu_crypto_pkt_process(ss, mb, num, tun_process); +} + +uint16_t +esp_inb_trs_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + return esp_inb_cpu_crypto_pkt_process(ss, mb, num, trs_process); +} + /* * process group of ESP inbound tunnel packets. */ diff --git a/lib/librte_ipsec/esp_outb.c b/lib/librte_ipsec/esp_outb.c index 55799a867..ecfc4cd3f 100644 --- a/lib/librte_ipsec/esp_outb.c +++ b/lib/librte_ipsec/esp_outb.c @@ -104,7 +104,7 @@ outb_cop_prepare(struct rte_crypto_op *cop, static inline int32_t outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, - union sym_op_data *icv, uint8_t sqh_len) + _set_icv_f set_icv, void *icv_val, uint8_t sqh_len) { uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen; struct rte_mbuf *ml; @@ -177,8 +177,8 @@ outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, espt->pad_len = pdlen; espt->next_proto = sa->proto; - icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); - icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); + /* set icv va/pa value(s) */ + set_icv(icv_val, ml, pdofs); return clen; } @@ -189,14 +189,14 @@ outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, */ static inline void outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, - const union sym_op_data *icv) + uint8_t *icv_va, void *aad_buf) { uint32_t *psqh; struct aead_gcm_aad *aad; /* insert SQN.hi between ESP trailer and ICV */ if (sa->sqh_len != 0) { - psqh = (uint32_t *)(icv->va - sa->sqh_len); + psqh = (uint32_t *)(icv_va - sa->sqh_len); psqh[0] = sqn_hi32(sqc); } @@ -205,7 +205,7 @@ outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc, * right now we support only one AEAD algorithm: AES-GCM . */ if (sa->aad_len != 0) { - aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len); + aad = aad_buf; aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa)); } } @@ -242,11 +242,12 @@ esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], gen_iv(iv, sqc); /* try to update the packet itself */ - rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv, - sa->sqh_len); + rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], set_icv_va_pa, + (void *)&icv, sa->sqh_len); /* success, setup crypto op */ if (rc >= 0) { - outb_pkt_xprepare(sa, sqc, &icv); + outb_pkt_xprepare(sa, sqc, icv.va, + (void *)(icv.va + sa->icv_len)); lksd_none_cop_prepare(cop[k], cs, mb[i]); outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc); k++; @@ -270,7 +271,7 @@ esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], static inline int32_t outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb, - uint32_t l2len, uint32_t l3len, union sym_op_data *icv, + uint32_t l2len, uint32_t l3len, _set_icv_f set_icv, void *icv_val, uint8_t sqh_len) { uint8_t np; @@ -340,8 +341,7 @@ outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc, espt->pad_len = pdlen; espt->next_proto = np; - icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs); - icv->pa = rte_pktmbuf_iova_offset(ml, pdofs); + set_icv(icv_val, ml, pdofs); return clen; } @@ -381,11 +381,12 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], gen_iv(iv, sqc); /* try to update the packet itself */ - rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3, &icv, - sa->sqh_len); + rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3, + set_icv_va_pa, (void *)&icv, sa->sqh_len); /* success, setup crypto op */ if (rc >= 0) { - outb_pkt_xprepare(sa, sqc, &icv); + outb_pkt_xprepare(sa, sqc, icv.va, + (void *)(icv.va + sa->icv_len)); lksd_none_cop_prepare(cop[k], cs, mb[i]); outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc); k++; @@ -403,6 +404,335 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], return k; } + +static inline int +outb_cpu_crypto_proc_prepare(struct rte_mbuf *m, const struct rte_ipsec_sa *sa, + uint32_t hlen, uint32_t plen, + struct rte_security_vec *buf, struct iovec *cur_vec, void *iv) +{ + struct rte_mbuf *ms; + uint64_t *ivp = iv; + struct aead_gcm_iv *gcm; + struct aesctr_cnt_blk *ctr; + struct iovec *vec = cur_vec; + uint32_t left; + uint32_t off = 0; + uint32_t n_seg = 0; + uint32_t algo; + + algo = sa->algo_type; + + switch (algo) { + case ALGO_TYPE_AES_GCM: + gcm = iv; + aead_gcm_iv_fill(gcm, ivp[0], sa->salt); + off = sa->ctp.cipher.offset + hlen; + left = sa->ctp.cipher.length + plen; + break; + case ALGO_TYPE_AES_CBC: + case ALGO_TYPE_3DES_CBC: + off = sa->ctp.auth.offset + hlen; + left = sa->ctp.auth.length + plen; + break; + case ALGO_TYPE_AES_CTR: + off = sa->ctp.auth.offset + hlen; + left = sa->ctp.auth.length + plen; + ctr = iv; + aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt); + break; + case ALGO_TYPE_NULL: + left = sa->ctp.cipher.length + plen; + break; + default: + return -EINVAL; + } + + ms = mbuf_get_seg_ofs(m, &off); + if (!ms) + return -1; + + while (n_seg < m->nb_segs && left && ms) { + uint32_t len = RTE_MIN(left, ms->data_len - off); + + vec->iov_base = rte_pktmbuf_mtod_offset(ms, void *, off); + vec->iov_len = len; + + left -= len; + vec++; + n_seg++; + ms = ms->next; + off = 0; + } + + if (left) + return -1; + + buf->vec = cur_vec; + buf->num = n_seg; + + return n_seg; +} + +static uint16_t +esp_outb_tun_cpu_crypto_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + struct rte_security_ctx *ctx; + struct rte_security_session *rss; + void *icv_va; + uint32_t dr[num]; + uint32_t i, n; + int32_t rc; + + /* cpu crypto specific variables */ + struct rte_security_vec buf[num]; + struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num]; + uint32_t vec_idx = 0; + uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD]; + void *iv[num]; + int status[num]; + uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)]; + void *aad[num]; + void *digest[num]; + uint32_t k; + + sa = ss->sa; + ctx = ss->security.ctx; + rss = ss->security.ses; + + k = 0; + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + for (i = 0; i != n; i++) { + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv_buf[k], sqc); + + /* try to update the packet itself */ + rc = outb_tun_pkt_prepare(sa, sqc, iv_buf[k], mb[i], set_icv_va, + (void *)&icv_va, sa->sqh_len); + + /* success, setup crypto op */ + if (rc >= 0) { + iv[k] = (void *)iv_buf[k]; + aad[k] = (void *)aad_buf[k]; + digest[k] = (void *)icv_va; + + outb_pkt_xprepare(sa, sqc, icv_va, aad[k]); + + rc = outb_cpu_crypto_proc_prepare(mb[i], sa, + 0, rc, &buf[k], &vec[vec_idx], iv[k]); + if (rc < 0) { + dr[i - k] = i; + rte_errno = -rc; + continue; + } + + vec_idx += rc; + k++; + /* failure, put packet into the death-row */ + } else { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not prepared mbufs beyond good ones */ + if (k != n && k != 0) + move_bad_mbufs(mb, dr, n, n - k); + + if (unlikely(k == 0)) { + rte_errno = EBADMSG; + return 0; + } + + /* process the packets */ + n = 0; + rc = rte_security_process_cpu_crypto_bulk(ctx, rss, buf, iv, aad, + digest, status, k); + /* move failed process packets to dr */ + if (rc < 0) + for (i = 0; i < n; i++) { + if (status[i]) + dr[n++] = i; + } + + if (n) + move_bad_mbufs(mb, dr, k, n); + + return k - n; +} + +static uint16_t +esp_outb_trs_cpu_crypto_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) + +{ + uint64_t sqn; + rte_be64_t sqc; + struct rte_ipsec_sa *sa; + struct rte_security_ctx *ctx; + struct rte_security_session *rss; + void *icv_va; + uint32_t dr[num]; + uint32_t i, n; + uint32_t l2, l3; + int32_t rc; + + /* cpu crypto specific variables */ + struct rte_security_vec buf[num]; + struct iovec vec[RTE_LIBRTE_IP_FRAG_MAX_FRAG * num]; + uint32_t vec_idx = 0; + uint64_t iv_buf[num][IPSEC_MAX_IV_QWORD]; + void *iv[num]; + int status[num]; + uint8_t *aad_buf[num][sizeof(struct aead_gcm_aad)]; + void *aad[num]; + void *digest[num]; + uint32_t k; + + sa = ss->sa; + ctx = ss->security.ctx; + rss = ss->security.ses; + + k = 0; + n = num; + sqn = esn_outb_update_sqn(sa, &n); + if (n != num) + rte_errno = EOVERFLOW; + + for (i = 0; i != n; i++) { + l2 = mb[i]->l2_len; + l3 = mb[i]->l3_len; + + sqc = rte_cpu_to_be_64(sqn + i); + gen_iv(iv_buf[k], sqc); + + /* try to update the packet itself */ + rc = outb_trs_pkt_prepare(sa, sqc, iv_buf[k], mb[i], l2, l3, + set_icv_va, (void *)&icv_va, sa->sqh_len); + + /* success, setup crypto op */ + if (rc >= 0) { + iv[k] = (void *)iv_buf[k]; + aad[k] = (void *)aad_buf[k]; + digest[k] = (void *)icv_va; + + outb_pkt_xprepare(sa, sqc, icv_va, aad[k]); + + rc = outb_cpu_crypto_proc_prepare(mb[i], sa, + l2 + l3, rc, &buf[k], &vec[vec_idx], + iv[k]); + if (rc < 0) { + dr[i - k] = i; + rte_errno = -rc; + continue; + } + + vec_idx += rc; + k++; + /* failure, put packet into the death-row */ + } else { + dr[i - k] = i; + rte_errno = -rc; + } + } + + /* copy not prepared mbufs beyond good ones */ + if (k != n && k != 0) + move_bad_mbufs(mb, dr, n, n - k); + + /* process the packets */ + n = 0; + rc = rte_security_process_cpu_crypto_bulk(ctx, rss, buf, iv, aad, + digest, status, k); + /* move failed process packets to dr */ + if (rc < 0) + for (i = 0; i < k; i++) { + if (status[i]) + dr[n++] = i; + } + + if (n) + move_bad_mbufs(mb, dr, k, n); + + return k - n; +} + +uint16_t +esp_outb_tun_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + struct rte_ipsec_sa *sa = ss->sa; + uint32_t icv_len; + void *icv; + uint16_t n; + uint16_t i; + + n = esp_outb_tun_cpu_crypto_process(ss, mb, num); + + icv_len = sa->icv_len; + + for (i = 0; i < n; i++) { + struct rte_mbuf *ml = rte_pktmbuf_lastseg(mb[i]); + + mb[i]->pkt_len -= sa->sqh_len; + ml->data_len -= sa->sqh_len; + + icv = rte_pktmbuf_mtod_offset(ml, void *, + ml->data_len - icv_len); + remove_sqh(icv, sa->icv_len); + } + + return n; +} + +uint16_t +esp_outb_tun_cpu_crypto_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + return esp_outb_tun_cpu_crypto_process(ss, mb, num); +} + +uint16_t +esp_outb_trs_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + struct rte_ipsec_sa *sa = ss->sa; + uint32_t icv_len; + void *icv; + uint16_t n; + uint16_t i; + + n = esp_outb_trs_cpu_crypto_process(ss, mb, num); + icv_len = sa->icv_len; + + for (i = 0; i < n; i++) { + struct rte_mbuf *ml = rte_pktmbuf_lastseg(mb[i]); + + mb[i]->pkt_len -= sa->sqh_len; + ml->data_len -= sa->sqh_len; + + icv = rte_pktmbuf_mtod_offset(ml, void *, + ml->data_len - icv_len); + remove_sqh(icv, sa->icv_len); + } + + return n; +} + +uint16_t +esp_outb_trs_cpu_crypto_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) +{ + return esp_outb_trs_cpu_crypto_process(ss, mb, num); +} + /* * process outbound packets for SA with ESN support, * for algorithms that require SQN.hibits to be implictly included @@ -410,8 +740,8 @@ esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], * In that case we have to move ICV bytes back to their proper place. */ uint16_t -esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - uint16_t num) +esp_outb_sqh_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) { uint32_t i, k, icv_len, *icv; struct rte_mbuf *ml; @@ -498,7 +828,8 @@ inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss, gen_iv(iv, sqc); /* try to update the packet itself */ - rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0); + rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], set_icv_va_pa, + (void *)&icv, 0); k += (rc >= 0); @@ -552,7 +883,7 @@ inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss, /* try to update the packet itself */ rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], - l2, l3, &icv, 0); + l2, l3, set_icv_va_pa, (void *)&icv, 0); k += (rc >= 0); diff --git a/lib/librte_ipsec/sa.c b/lib/librte_ipsec/sa.c index 23d394b46..b8d55a1c7 100644 --- a/lib/librte_ipsec/sa.c +++ b/lib/librte_ipsec/sa.c @@ -544,9 +544,9 @@ lksd_proto_prepare(const struct rte_ipsec_session *ss, * - inbound/outbound for RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL * - outbound for RTE_SECURITY_ACTION_TYPE_NONE when ESN is disabled */ -static uint16_t -pkt_flag_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], - uint16_t num) +uint16_t +esp_outb_pkt_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num) { uint32_t i, k; uint32_t dr[num]; @@ -599,12 +599,48 @@ lksd_none_pkt_func_select(const struct rte_ipsec_sa *sa, case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6): pf->prepare = esp_outb_tun_prepare; pf->process = (sa->sqh_len != 0) ? - esp_outb_sqh_process : pkt_flag_process; + esp_outb_sqh_process : esp_outb_pkt_flag_process; break; case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS): pf->prepare = esp_outb_trs_prepare; pf->process = (sa->sqh_len != 0) ? - esp_outb_sqh_process : pkt_flag_process; + esp_outb_sqh_process : esp_outb_pkt_flag_process; + break; + default: + rc = -ENOTSUP; + } + + return rc; +} + +static int +cpu_crypto_pkt_func_select(const struct rte_ipsec_sa *sa, + struct rte_ipsec_sa_pkt_func *pf) +{ + int32_t rc; + + static const uint64_t msk = RTE_IPSEC_SATP_DIR_MASK | + RTE_IPSEC_SATP_MODE_MASK; + + rc = 0; + switch (sa->type & msk) { + case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4): + case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6): + pf->process = esp_inb_tun_cpu_crypto_pkt_process; + break; + case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS): + pf->process = esp_inb_trs_cpu_crypto_pkt_process; + break; + case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4): + case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6): + pf->process = (sa->sqh_len != 0) ? + esp_outb_tun_cpu_crypto_sqh_process : + esp_outb_tun_cpu_crypto_flag_process; + break; + case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS): + pf->process = (sa->sqh_len != 0) ? + esp_outb_trs_cpu_crypto_sqh_process : + esp_outb_trs_cpu_crypto_flag_process; break; default: rc = -ENOTSUP; @@ -672,13 +708,16 @@ ipsec_sa_pkt_func_select(const struct rte_ipsec_session *ss, case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL: if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) - pf->process = pkt_flag_process; + pf->process = esp_outb_pkt_flag_process; else pf->process = inline_proto_outb_pkt_process; break; case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL: pf->prepare = lksd_proto_prepare; - pf->process = pkt_flag_process; + pf->process = esp_outb_pkt_flag_process; + break; + case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO: + rc = cpu_crypto_pkt_func_select(sa, pf); break; default: rc = -ENOTSUP; diff --git a/lib/librte_ipsec/sa.h b/lib/librte_ipsec/sa.h index 51e69ad05..770d36b8b 100644 --- a/lib/librte_ipsec/sa.h +++ b/lib/librte_ipsec/sa.h @@ -156,6 +156,14 @@ uint16_t inline_inb_trs_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], uint16_t num); +uint16_t +esp_inb_tun_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +esp_inb_trs_cpu_crypto_pkt_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + /* outbound processing */ uint16_t @@ -170,6 +178,10 @@ uint16_t esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], uint16_t num); +uint16_t +esp_outb_pkt_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + uint16_t inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], uint16_t num); @@ -182,4 +194,21 @@ uint16_t inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[], uint16_t num); +uint16_t +esp_outb_tun_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +esp_outb_tun_cpu_crypto_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +esp_outb_trs_cpu_crypto_sqh_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + +uint16_t +esp_outb_trs_cpu_crypto_flag_process(const struct rte_ipsec_session *ss, + struct rte_mbuf *mb[], uint16_t num); + + #endif /* _SA_H_ */ diff --git a/lib/librte_ipsec/ses.c b/lib/librte_ipsec/ses.c index 82c765a33..eaa8c17b7 100644 --- a/lib/librte_ipsec/ses.c +++ b/lib/librte_ipsec/ses.c @@ -19,7 +19,9 @@ session_check(struct rte_ipsec_session *ss) return -EINVAL; if ((ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO || ss->type == - RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) && + RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL || + ss->type == + RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) && ss->security.ctx == NULL) return -EINVAL; } -- 2.14.5