* [dpdk-dev] [PATCH] port: add sym crypto port @ 2018-08-20 11:38 Zhang, Roy Fan 2018-08-20 14:29 ` [dpdk-dev] [PATCH v2] " Zhang, Roy Fan 0 siblings, 1 reply; 6+ messages in thread From: Zhang, Roy Fan @ 2018-08-20 11:38 UTC (permalink / raw) To: dev; +Cc: cristian.dumitrescu This patch adds the symmetric crypto support to port library. The crypto port acts as a shim layer to DPDK cryptodev library and supports in-place crypto workload processing. Change-Id: I8e9c87bdaaa252b39873596cbfaa2de2324e765c Signed-off-by: Zhang, Roy Fan <roy.fan.zhang@intel.com> --- lib/librte_port/Makefile | 2 + lib/librte_port/rte_port_sym_crypto.c | 552 ++++++++++++++++++++++++++++++++++ lib/librte_port/rte_port_sym_crypto.h | 93 ++++++ 3 files changed, 647 insertions(+) create mode 100644 lib/librte_port/rte_port_sym_crypto.c create mode 100644 lib/librte_port/rte_port_sym_crypto.h diff --git a/lib/librte_port/Makefile b/lib/librte_port/Makefile index 8df4864ed..30113f790 100644 --- a/lib/librte_port/Makefile +++ b/lib/librte_port/Makefile @@ -38,6 +38,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c endif SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_source_sink.c +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sym_crypto.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port.h @@ -53,5 +54,6 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sym_crypto.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_port/rte_port_sym_crypto.c b/lib/librte_port/rte_port_sym_crypto.c new file mode 100644 index 000000000..1cd242e22 --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.c @@ -0,0 +1,552 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <string.h> + +#include <rte_common.h> +#include <rte_malloc.h> + +#include "rte_port_sym_crypto.h" + +/* + * Port Crypto Reader + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_reader { + struct rte_port_in_stats stats; + + uint8_t cryptodev_id; + uint16_t queue_id; + struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; + rte_port_sym_crypto_reader_callback_fn f_callback; + void *arg_callback; +}; + +static void * +rte_port_sym_crypto_reader_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_reader_params *conf = + params; + struct rte_port_sym_crypto_reader *port; + + /* Check input parameters */ + if (conf == NULL) { + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->f_callback = conf->f_callback; + port->arg_callback = conf->arg_callback; + + return port; +} + +static int +rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) +{ + struct rte_port_sym_crypto_reader *p = + port; + uint16_t rx_ops_cnt, i, n = 0; + + rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id, + p->ops, n_pkts); + + for (i = 0; i < rx_ops_cnt; i++) { + struct rte_crypto_op *op = p->ops[i]; + + /** Drop failed pkts */ + if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) { + rte_pktmbuf_free(op->sym->m_src); + continue; + } + + pkts[n++] = op->sym->m_src; + } + + if (p->f_callback) + (*p->f_callback)(pkts, n, p->arg_callback); + + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_pkt_cnt - n); + + return n; +} + +static int +rte_port_sym_crypto_reader_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); + return -EINVAL; + } + + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_reader_stats_read(void *port, + struct rte_port_in_stats *stats, int clear) +{ + struct rte_port_sym_crypto_reader *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_params *conf = + params; + struct rte_port_sym_crypto_writer *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + + return port; +} + +static inline void +send_burst(struct rte_port_sym_crypto_writer *p) +{ + uint32_t nb_tx; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - + nb_tx); + for (; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer *p = + port; + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } else { + for (; pkts_mask;) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_flush(void *port) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (p->tx_buf_count > 0) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer Nodrop + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer_nodrop { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + uint64_t n_retries; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_nodrop_params *conf = + params; + struct rte_port_sym_crypto_writer_nodrop *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + /* + * When n_retries is 0 it means that we should wait for every packet to + * send no matter how many retries should it take. To limit number of + * branches in fast path, we use UINT64_MAX instead of branching. + */ + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; + + return port; +} + +static inline void +send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) +{ + uint32_t nb_tx = 0, i; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + /* We sent all the packets in a first try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + + for (i = 0; i < p->n_retries; i++) { + nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, + p->queue_id, p->tx_buf + nb_tx, + p->tx_buf_count - nb_tx); + + /* We sent all the packets in more than one try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + } + + /* We didn't send the packets in maximum allowed attempts */ + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, + p->tx_buf_count - nb_tx); + for ( ; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } else { + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, + 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_flush(void *port) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (p->tx_buf_count > 0) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_nodrop_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + + +/* + * Summary of port operations + */ +struct rte_port_in_ops rte_port_sym_crypto_reader_ops = { + .f_create = rte_port_sym_crypto_reader_create, + .f_free = rte_port_sym_crypto_reader_free, + .f_rx = rte_port_sym_crypto_reader_rx, + .f_stats = rte_port_sym_crypto_reader_stats_read, +}; + +struct rte_port_out_ops rte_port_sym_crypto_writer_ops = { + .f_create = rte_port_sym_crypto_writer_create, + .f_free = rte_port_sym_crypto_writer_free, + .f_tx = rte_port_sym_crypto_writer_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_flush, + .f_stats = rte_port_sym_crypto_writer_stats_read, +}; + +struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = { + .f_create = rte_port_sym_crypto_writer_nodrop_create, + .f_free = rte_port_sym_crypto_writer_nodrop_free, + .f_tx = rte_port_sym_crypto_writer_nodrop_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_nodrop_flush, + .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, +}; diff --git a/lib/librte_port/rte_port_sym_crypto.h b/lib/librte_port/rte_port_sym_crypto.h new file mode 100644 index 000000000..db66f1540 --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ +#define __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Port sym crypto Interface + * + * crypto_reader: input port built on top of pre-initialized crypto interface + * crypto_writer: output port built on top of pre-initialized crypto interface + * + ***/ + +#include <stdint.h> + +#include <rte_cryptodev.h> + +#include "rte_port.h" + +/** Function prototype for reader post action. */ +typedef void (*rte_port_sym_crypto_reader_callback_fn)(struct rte_mbuf **pkts, + uint16_t n_pkts, void *arg); + +/** Crypto_reader port parameters */ +struct rte_port_sym_crypto_reader_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** Crypto reader post callback function. */ + rte_port_sym_crypto_reader_callback_fn f_callback; + + /** Crypto reader post callback function arguments. */ + void *arg_callback; +}; + +/** Crypto_reader port operations. */ +extern struct rte_port_in_ops rte_port_sym_crypto_reader_ops; + + +/** Crypto_writer port parameters. */ +struct rte_port_sym_crypto_writer_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; +}; + +/** Crypto_writer port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_ops; + +/** Crypto_writer_nodrop port parameters. */ +struct rte_port_sym_crypto_writer_nodrop_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev queue pair id. */ + uint16_t queue_id; + + /** Offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; + + /** Maximum number of retries, 0 for no limit. */ + uint32_t n_retries; +}; + +/** Crypto_writer_nodrop port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops; + +#ifdef __cplusplus +} +#endif + +#endif -- 2.13.6 ^ permalink raw reply [flat|nested] 6+ messages in thread
* [dpdk-dev] [PATCH v2] port: add sym crypto port 2018-08-20 11:38 [dpdk-dev] [PATCH] port: add sym crypto port Zhang, Roy Fan @ 2018-08-20 14:29 ` Zhang, Roy Fan 2018-09-28 10:41 ` [dpdk-dev] [PATCH v3] " Fan Zhang 0 siblings, 1 reply; 6+ messages in thread From: Zhang, Roy Fan @ 2018-08-20 14:29 UTC (permalink / raw) To: dev; +Cc: cristian.dumitrescu, Fan Zhang This patch adds the symmetric crypto support to port library. The crypto port acts as a shim layer to DPDK cryptodev library and supports in-place crypto workload processing. Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com> Acked-by: Dumitrescu, Cristian <cristian.dumitrescu@intel.com> --- v2: - added experimental flags - enabled meson build lib/librte_port/Makefile | 2 + lib/librte_port/meson.build | 8 +- lib/librte_port/rte_port_sym_crypto.c | 553 ++++++++++++++++++++++++++++++++++ lib/librte_port/rte_port_sym_crypto.h | 97 ++++++ lib/librte_port/rte_port_version.map | 8 + 5 files changed, 665 insertions(+), 3 deletions(-) create mode 100644 lib/librte_port/rte_port_sym_crypto.c create mode 100644 lib/librte_port/rte_port_sym_crypto.h diff --git a/lib/librte_port/Makefile b/lib/librte_port/Makefile index 8df4864ed..30113f790 100644 --- a/lib/librte_port/Makefile +++ b/lib/librte_port/Makefile @@ -38,6 +38,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c endif SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_source_sink.c +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sym_crypto.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port.h @@ -53,5 +54,6 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sym_crypto.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_port/meson.build b/lib/librte_port/meson.build index f3d8b4434..0d11456f0 100644 --- a/lib/librte_port/meson.build +++ b/lib/librte_port/meson.build @@ -9,7 +9,8 @@ sources = files( 'rte_port_ras.c', 'rte_port_ring.c', 'rte_port_sched.c', - 'rte_port_source_sink.c') + 'rte_port_source_sink.c', + 'rte_port_sym_crypto.c') headers = files( 'rte_port_ethdev.h', 'rte_port_fd.h', @@ -18,8 +19,9 @@ headers = files( 'rte_port.h', 'rte_port_ring.h', 'rte_port_sched.h', - 'rte_port_source_sink.h') -deps += ['ethdev', 'sched', 'ip_frag'] + 'rte_port_source_sink.h', + 'rte_port_sym_crypto.h') +deps += ['ethdev', 'sched', 'ip_frag', 'cryptodev'] if dpdk_conf.has('RTE_LIBRTE_KNI') sources += files('rte_port_kni.c') diff --git a/lib/librte_port/rte_port_sym_crypto.c b/lib/librte_port/rte_port_sym_crypto.c new file mode 100644 index 000000000..aa3de4fdf --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.c @@ -0,0 +1,553 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <string.h> + +#include <rte_common.h> +#include <rte_malloc.h> + +#include "rte_port_sym_crypto.h" + +/* + * Port Crypto Reader + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_reader { + struct rte_port_in_stats stats; + + uint8_t cryptodev_id; + uint16_t queue_id; + struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; + rte_port_sym_crypto_reader_callback_fn f_callback; + void *arg_callback; +}; + +static void * +rte_port_sym_crypto_reader_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_reader_params *conf = + params; + struct rte_port_sym_crypto_reader *port; + + /* Check input parameters */ + if (conf == NULL) { + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->f_callback = conf->f_callback; + port->arg_callback = conf->arg_callback; + + return port; +} + +static int +rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) +{ + struct rte_port_sym_crypto_reader *p = + port; + uint16_t rx_ops_cnt, i, n = 0; + + rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id, + p->ops, n_pkts); + + for (i = 0; i < rx_ops_cnt; i++) { + struct rte_crypto_op *op = p->ops[i]; + + /** Drop failed pkts */ + if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) { + rte_pktmbuf_free(op->sym->m_src); + continue; + } + + pkts[n++] = op->sym->m_src; + } + + if (p->f_callback) + (*p->f_callback)(pkts, n, p->arg_callback); + + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_pkt_cnt - n); + + return n; +} + +static int +rte_port_sym_crypto_reader_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); + return -EINVAL; + } + + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_reader_stats_read(void *port, + struct rte_port_in_stats *stats, int clear) +{ + struct rte_port_sym_crypto_reader *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_params *conf = + params; + struct rte_port_sym_crypto_writer *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + + return port; +} + +static inline void +send_burst(struct rte_port_sym_crypto_writer *p) +{ + uint32_t nb_tx; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - + nb_tx); + for (; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer *p = + port; + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } else { + for (; pkts_mask;) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_flush(void *port) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (p->tx_buf_count > 0) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer Nodrop + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer_nodrop { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + uint64_t n_retries; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_nodrop_params *conf = + params; + struct rte_port_sym_crypto_writer_nodrop *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + /* + * When n_retries is 0 it means that we should wait for every packet to + * send no matter how many retries should it take. To limit number of + * branches in fast path, we use UINT64_MAX instead of branching. + */ + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; + + return port; +} + +static inline void +send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) +{ + uint32_t nb_tx = 0, i; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + /* We sent all the packets in a first try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + + for (i = 0; i < p->n_retries; i++) { + nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, + p->queue_id, p->tx_buf + nb_tx, + p->tx_buf_count - nb_tx); + + /* We sent all the packets in more than one try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + } + + /* We didn't send the packets in maximum allowed attempts */ + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, + p->tx_buf_count - nb_tx); + for ( ; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } else { + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, + 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_flush(void *port) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (p->tx_buf_count > 0) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_nodrop_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + + +/* + * Summary of port operations + */ +struct rte_port_in_ops __rte_experimental rte_port_sym_crypto_reader_ops = { + .f_create = rte_port_sym_crypto_reader_create, + .f_free = rte_port_sym_crypto_reader_free, + .f_rx = rte_port_sym_crypto_reader_rx, + .f_stats = rte_port_sym_crypto_reader_stats_read, +}; + +struct rte_port_out_ops __rte_experimental rte_port_sym_crypto_writer_ops = { + .f_create = rte_port_sym_crypto_writer_create, + .f_free = rte_port_sym_crypto_writer_free, + .f_tx = rte_port_sym_crypto_writer_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_flush, + .f_stats = rte_port_sym_crypto_writer_stats_read, +}; + +struct rte_port_out_ops __rte_experimental +rte_port_sym_crypto_writer_nodrop_ops = { + .f_create = rte_port_sym_crypto_writer_nodrop_create, + .f_free = rte_port_sym_crypto_writer_nodrop_free, + .f_tx = rte_port_sym_crypto_writer_nodrop_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_nodrop_flush, + .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, +}; diff --git a/lib/librte_port/rte_port_sym_crypto.h b/lib/librte_port/rte_port_sym_crypto.h new file mode 100644 index 000000000..daf4a36dd --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ +#define __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Port sym crypto Interface + * + * crypto_reader: input port built on top of pre-initialized crypto interface + * crypto_writer: output port built on top of pre-initialized crypto interface + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + ***/ + +#include <stdint.h> + +#include <rte_cryptodev.h> + +#include "rte_port.h" + +/** Function prototype for reader post action. */ +typedef void (*rte_port_sym_crypto_reader_callback_fn)(struct rte_mbuf **pkts, + uint16_t n_pkts, void *arg); + +/** Crypto_reader port parameters */ +struct rte_port_sym_crypto_reader_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** Crypto reader post callback function. */ + rte_port_sym_crypto_reader_callback_fn f_callback; + + /** Crypto reader post callback function arguments. */ + void *arg_callback; +}; + +/** Crypto_reader port operations. */ +extern struct rte_port_in_ops __rte_experimental +rte_port_sym_crypto_reader_ops; + + +/** Crypto_writer port parameters. */ +struct rte_port_sym_crypto_writer_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; +}; + +/** Crypto_writer port operations. */ +extern struct rte_port_out_ops __rte_experimental +rte_port_sym_crypto_writer_ops; + +/** Crypto_writer_nodrop port parameters. */ +struct rte_port_sym_crypto_writer_nodrop_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev queue pair id. */ + uint16_t queue_id; + + /** Offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; + + /** Maximum number of retries, 0 for no limit. */ + uint32_t n_retries; +}; + +/** Crypto_writer_nodrop port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/librte_port/rte_port_version.map b/lib/librte_port/rte_port_version.map index 6470629b4..20e45a0ef 100644 --- a/lib/librte_port/rte_port_version.map +++ b/lib/librte_port/rte_port_version.map @@ -51,3 +51,11 @@ DPDK_16.11 { rte_port_fd_writer_nodrop_ops; } DPDK_16.07; + +EXPERIMENTAL { + global: + + rte_port_sym_crypto_reader_ops; + rte_port_sym_crypto_writer_ops; + rte_port_sym_crypto_writer_nodrop_ops; +}; -- 2.13.6 ^ permalink raw reply [flat|nested] 6+ messages in thread
* [dpdk-dev] [PATCH v3] port: add sym crypto port 2018-08-20 14:29 ` [dpdk-dev] [PATCH v2] " Zhang, Roy Fan @ 2018-09-28 10:41 ` Fan Zhang 2018-09-28 11:32 ` Dumitrescu, Cristian 2018-10-01 10:02 ` [dpdk-dev] [PATCH v4] " Fan Zhang 0 siblings, 2 replies; 6+ messages in thread From: Fan Zhang @ 2018-09-28 10:41 UTC (permalink / raw) To: dev; +Cc: cristian.dumitrescu, Zhang, Roy Fan, Zhang From: "Zhang, Roy Fan" <roy.fan.zhang@intel.com> This patch adds the symmetric crypto support to port library. The crypto port acts as a shim layer to DPDK cryptodev library and supports in-place crypto workload processing. Signed-off-by: Zhang, Roy Fan <roy.fan.zhang@intel.com> Acked-by: Dumitrescu, Cristian <cristian.dumitrescu@intel.com> --- v3: - fixed a compile issue v2: - added experimental flags - enabled meson build lib/librte_port/Makefile | 4 +- lib/librte_port/meson.build | 8 +- lib/librte_port/rte_port_sym_crypto.c | 553 ++++++++++++++++++++++++++++++++++ lib/librte_port/rte_port_sym_crypto.h | 97 ++++++ lib/librte_port/rte_port_version.map | 8 + 5 files changed, 666 insertions(+), 4 deletions(-) create mode 100644 lib/librte_port/rte_port_sym_crypto.c create mode 100644 lib/librte_port/rte_port_sym_crypto.h diff --git a/lib/librte_port/Makefile b/lib/librte_port/Makefile index 8df4864ed..1b83f6f2b 100644 --- a/lib/librte_port/Makefile +++ b/lib/librte_port/Makefile @@ -11,7 +11,7 @@ ifeq ($(CONFIG_RTE_PORT_PCAP),y) LDLIBS += -lpcap endif LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev -LDLIBS += -lrte_ip_frag -lrte_sched +LDLIBS += -lrte_ip_frag -lrte_sched -lrte_cryptodev ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) LDLIBS += -lrte_kni endif @@ -38,6 +38,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c endif SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_source_sink.c +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sym_crypto.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port.h @@ -53,5 +54,6 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sym_crypto.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_port/meson.build b/lib/librte_port/meson.build index f3d8b4434..0d11456f0 100644 --- a/lib/librte_port/meson.build +++ b/lib/librte_port/meson.build @@ -9,7 +9,8 @@ sources = files( 'rte_port_ras.c', 'rte_port_ring.c', 'rte_port_sched.c', - 'rte_port_source_sink.c') + 'rte_port_source_sink.c', + 'rte_port_sym_crypto.c') headers = files( 'rte_port_ethdev.h', 'rte_port_fd.h', @@ -18,8 +19,9 @@ headers = files( 'rte_port.h', 'rte_port_ring.h', 'rte_port_sched.h', - 'rte_port_source_sink.h') -deps += ['ethdev', 'sched', 'ip_frag'] + 'rte_port_source_sink.h', + 'rte_port_sym_crypto.h') +deps += ['ethdev', 'sched', 'ip_frag', 'cryptodev'] if dpdk_conf.has('RTE_LIBRTE_KNI') sources += files('rte_port_kni.c') diff --git a/lib/librte_port/rte_port_sym_crypto.c b/lib/librte_port/rte_port_sym_crypto.c new file mode 100644 index 000000000..aa3de4fdf --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.c @@ -0,0 +1,553 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <string.h> + +#include <rte_common.h> +#include <rte_malloc.h> + +#include "rte_port_sym_crypto.h" + +/* + * Port Crypto Reader + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_reader { + struct rte_port_in_stats stats; + + uint8_t cryptodev_id; + uint16_t queue_id; + struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; + rte_port_sym_crypto_reader_callback_fn f_callback; + void *arg_callback; +}; + +static void * +rte_port_sym_crypto_reader_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_reader_params *conf = + params; + struct rte_port_sym_crypto_reader *port; + + /* Check input parameters */ + if (conf == NULL) { + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->f_callback = conf->f_callback; + port->arg_callback = conf->arg_callback; + + return port; +} + +static int +rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) +{ + struct rte_port_sym_crypto_reader *p = + port; + uint16_t rx_ops_cnt, i, n = 0; + + rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id, + p->ops, n_pkts); + + for (i = 0; i < rx_ops_cnt; i++) { + struct rte_crypto_op *op = p->ops[i]; + + /** Drop failed pkts */ + if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) { + rte_pktmbuf_free(op->sym->m_src); + continue; + } + + pkts[n++] = op->sym->m_src; + } + + if (p->f_callback) + (*p->f_callback)(pkts, n, p->arg_callback); + + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_pkt_cnt - n); + + return n; +} + +static int +rte_port_sym_crypto_reader_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); + return -EINVAL; + } + + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_reader_stats_read(void *port, + struct rte_port_in_stats *stats, int clear) +{ + struct rte_port_sym_crypto_reader *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_params *conf = + params; + struct rte_port_sym_crypto_writer *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + + return port; +} + +static inline void +send_burst(struct rte_port_sym_crypto_writer *p) +{ + uint32_t nb_tx; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - + nb_tx); + for (; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer *p = + port; + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } else { + for (; pkts_mask;) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_flush(void *port) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (p->tx_buf_count > 0) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer Nodrop + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer_nodrop { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + uint64_t n_retries; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_nodrop_params *conf = + params; + struct rte_port_sym_crypto_writer_nodrop *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + /* + * When n_retries is 0 it means that we should wait for every packet to + * send no matter how many retries should it take. To limit number of + * branches in fast path, we use UINT64_MAX instead of branching. + */ + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; + + return port; +} + +static inline void +send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) +{ + uint32_t nb_tx = 0, i; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + /* We sent all the packets in a first try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + + for (i = 0; i < p->n_retries; i++) { + nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, + p->queue_id, p->tx_buf + nb_tx, + p->tx_buf_count - nb_tx); + + /* We sent all the packets in more than one try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + } + + /* We didn't send the packets in maximum allowed attempts */ + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, + p->tx_buf_count - nb_tx); + for ( ; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } else { + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, + 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_flush(void *port) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (p->tx_buf_count > 0) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_nodrop_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + + +/* + * Summary of port operations + */ +struct rte_port_in_ops __rte_experimental rte_port_sym_crypto_reader_ops = { + .f_create = rte_port_sym_crypto_reader_create, + .f_free = rte_port_sym_crypto_reader_free, + .f_rx = rte_port_sym_crypto_reader_rx, + .f_stats = rte_port_sym_crypto_reader_stats_read, +}; + +struct rte_port_out_ops __rte_experimental rte_port_sym_crypto_writer_ops = { + .f_create = rte_port_sym_crypto_writer_create, + .f_free = rte_port_sym_crypto_writer_free, + .f_tx = rte_port_sym_crypto_writer_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_flush, + .f_stats = rte_port_sym_crypto_writer_stats_read, +}; + +struct rte_port_out_ops __rte_experimental +rte_port_sym_crypto_writer_nodrop_ops = { + .f_create = rte_port_sym_crypto_writer_nodrop_create, + .f_free = rte_port_sym_crypto_writer_nodrop_free, + .f_tx = rte_port_sym_crypto_writer_nodrop_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_nodrop_flush, + .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, +}; diff --git a/lib/librte_port/rte_port_sym_crypto.h b/lib/librte_port/rte_port_sym_crypto.h new file mode 100644 index 000000000..daf4a36dd --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ +#define __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Port sym crypto Interface + * + * crypto_reader: input port built on top of pre-initialized crypto interface + * crypto_writer: output port built on top of pre-initialized crypto interface + * + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + ***/ + +#include <stdint.h> + +#include <rte_cryptodev.h> + +#include "rte_port.h" + +/** Function prototype for reader post action. */ +typedef void (*rte_port_sym_crypto_reader_callback_fn)(struct rte_mbuf **pkts, + uint16_t n_pkts, void *arg); + +/** Crypto_reader port parameters */ +struct rte_port_sym_crypto_reader_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** Crypto reader post callback function. */ + rte_port_sym_crypto_reader_callback_fn f_callback; + + /** Crypto reader post callback function arguments. */ + void *arg_callback; +}; + +/** Crypto_reader port operations. */ +extern struct rte_port_in_ops __rte_experimental +rte_port_sym_crypto_reader_ops; + + +/** Crypto_writer port parameters. */ +struct rte_port_sym_crypto_writer_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; +}; + +/** Crypto_writer port operations. */ +extern struct rte_port_out_ops __rte_experimental +rte_port_sym_crypto_writer_ops; + +/** Crypto_writer_nodrop port parameters. */ +struct rte_port_sym_crypto_writer_nodrop_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev queue pair id. */ + uint16_t queue_id; + + /** Offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; + + /** Maximum number of retries, 0 for no limit. */ + uint32_t n_retries; +}; + +/** Crypto_writer_nodrop port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/librte_port/rte_port_version.map b/lib/librte_port/rte_port_version.map index 6470629b4..20e45a0ef 100644 --- a/lib/librte_port/rte_port_version.map +++ b/lib/librte_port/rte_port_version.map @@ -51,3 +51,11 @@ DPDK_16.11 { rte_port_fd_writer_nodrop_ops; } DPDK_16.07; + +EXPERIMENTAL { + global: + + rte_port_sym_crypto_reader_ops; + rte_port_sym_crypto_writer_ops; + rte_port_sym_crypto_writer_nodrop_ops; +}; -- 2.13.6 ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [dpdk-dev] [PATCH v3] port: add sym crypto port 2018-09-28 10:41 ` [dpdk-dev] [PATCH v3] " Fan Zhang @ 2018-09-28 11:32 ` Dumitrescu, Cristian 2018-10-01 10:02 ` [dpdk-dev] [PATCH v4] " Fan Zhang 1 sibling, 0 replies; 6+ messages in thread From: Dumitrescu, Cristian @ 2018-09-28 11:32 UTC (permalink / raw) To: Zhang, Roy Fan, dev > -----Original Message----- > From: Zhang, Roy Fan > Sent: Friday, September 28, 2018 11:42 AM > To: dev@dpdk.org > Cc: Dumitrescu, Cristian <cristian.dumitrescu@intel.com>; Zhang, Roy Fan > <roy.fan.zhang@intel.com>; Zhang > Subject: [PATCH v3] port: add sym crypto port > > From: "Zhang, Roy Fan" <roy.fan.zhang@intel.com> > > This patch adds the symmetric crypto support to port library. > The crypto port acts as a shim layer to DPDK cryptodev library and > supports in-place crypto workload processing. > > Signed-off-by: Zhang, Roy Fan <roy.fan.zhang@intel.com> > Acked-by: Dumitrescu, Cristian <cristian.dumitrescu@intel.com> > --- > v3: > - fixed a compile issue > > v2: > - added experimental flags > - enabled meson build > > lib/librte_port/Makefile | 4 +- > lib/librte_port/meson.build | 8 +- > lib/librte_port/rte_port_sym_crypto.c | 553 > ++++++++++++++++++++++++++++++++++ > lib/librte_port/rte_port_sym_crypto.h | 97 ++++++ > lib/librte_port/rte_port_version.map | 8 + > 5 files changed, 666 insertions(+), 4 deletions(-) > create mode 100644 lib/librte_port/rte_port_sym_crypto.c > create mode 100644 lib/librte_port/rte_port_sym_crypto.h > > diff --git a/lib/librte_port/Makefile b/lib/librte_port/Makefile > index 8df4864ed..1b83f6f2b 100644 > --- a/lib/librte_port/Makefile > +++ b/lib/librte_port/Makefile > @@ -11,7 +11,7 @@ ifeq ($(CONFIG_RTE_PORT_PCAP),y) > LDLIBS += -lpcap > endif > LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev > -LDLIBS += -lrte_ip_frag -lrte_sched > +LDLIBS += -lrte_ip_frag -lrte_sched -lrte_cryptodev > ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) > LDLIBS += -lrte_kni > endif > @@ -38,6 +38,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) > SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c > endif > SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_source_sink.c > +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sym_crypto.c > > # install includes > SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port.h > @@ -53,5 +54,6 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) > SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h > endif > SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h > +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sym_crypto.h > > include $(RTE_SDK)/mk/rte.lib.mk > diff --git a/lib/librte_port/meson.build b/lib/librte_port/meson.build > index f3d8b4434..0d11456f0 100644 > --- a/lib/librte_port/meson.build > +++ b/lib/librte_port/meson.build > @@ -9,7 +9,8 @@ sources = files( > 'rte_port_ras.c', > 'rte_port_ring.c', > 'rte_port_sched.c', > - 'rte_port_source_sink.c') > + 'rte_port_source_sink.c', > + 'rte_port_sym_crypto.c') > headers = files( > 'rte_port_ethdev.h', > 'rte_port_fd.h', > @@ -18,8 +19,9 @@ headers = files( > 'rte_port.h', > 'rte_port_ring.h', > 'rte_port_sched.h', > - 'rte_port_source_sink.h') > -deps += ['ethdev', 'sched', 'ip_frag'] > + 'rte_port_source_sink.h', > + 'rte_port_sym_crypto.h') > +deps += ['ethdev', 'sched', 'ip_frag', 'cryptodev'] > > if dpdk_conf.has('RTE_LIBRTE_KNI') > sources += files('rte_port_kni.c') > diff --git a/lib/librte_port/rte_port_sym_crypto.c > b/lib/librte_port/rte_port_sym_crypto.c > new file mode 100644 > index 000000000..aa3de4fdf > --- /dev/null > +++ b/lib/librte_port/rte_port_sym_crypto.c > @@ -0,0 +1,553 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2018 Intel Corporation > + */ > +#include <string.h> > + > +#include <rte_common.h> > +#include <rte_malloc.h> > + > +#include "rte_port_sym_crypto.h" > + > +/* > + * Port Crypto Reader > + */ > +#ifdef RTE_PORT_STATS_COLLECT > + > +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) > \ > + port->stats.n_pkts_in += val > +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, > val) \ > + port->stats.n_pkts_drop += val > + > +#else > + > +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) > +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, > val) > + > +#endif > + > +struct rte_port_sym_crypto_reader { > + struct rte_port_in_stats stats; > + > + uint8_t cryptodev_id; > + uint16_t queue_id; > + struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; > + rte_port_sym_crypto_reader_callback_fn f_callback; > + void *arg_callback; > +}; > + > +static void * > +rte_port_sym_crypto_reader_create(void *params, int socket_id) > +{ > + struct rte_port_sym_crypto_reader_params *conf = > + params; > + struct rte_port_sym_crypto_reader *port; > + > + /* Check input parameters */ > + if (conf == NULL) { > + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); > + return NULL; > + } > + > + /* Memory allocation */ > + port = rte_zmalloc_socket("PORT", sizeof(*port), > + RTE_CACHE_LINE_SIZE, socket_id); > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", > __func__); > + return NULL; > + } > + > + /* Initialization */ > + port->cryptodev_id = conf->cryptodev_id; > + port->queue_id = conf->queue_id; > + port->f_callback = conf->f_callback; > + port->arg_callback = conf->arg_callback; > + > + return port; > +} > + > +static int > +rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, > uint32_t n_pkts) > +{ > + struct rte_port_sym_crypto_reader *p = > + port; > + uint16_t rx_ops_cnt, i, n = 0; > + > + rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p- > >queue_id, > + p->ops, n_pkts); > + > + for (i = 0; i < rx_ops_cnt; i++) { > + struct rte_crypto_op *op = p->ops[i]; > + > + /** Drop failed pkts */ > + if (unlikely(op->status != > RTE_CRYPTO_OP_STATUS_SUCCESS)) { > + rte_pktmbuf_free(op->sym->m_src); > + continue; > + } > + > + pkts[n++] = op->sym->m_src; > + } > + > + if (p->f_callback) > + (*p->f_callback)(pkts, n, p->arg_callback); > + > + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); > + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, > rx_pkt_cnt - n); > + > + return n; > +} > + > +static int > +rte_port_sym_crypto_reader_free(void *port) > +{ > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); > + return -EINVAL; > + } > + > + rte_free(port); > + > + return 0; > +} > + > +static int rte_port_sym_crypto_reader_stats_read(void *port, > + struct rte_port_in_stats *stats, int clear) > +{ > + struct rte_port_sym_crypto_reader *p = > + port; > + > + if (stats != NULL) > + memcpy(stats, &p->stats, sizeof(p->stats)); > + > + if (clear) > + memset(&p->stats, 0, sizeof(p->stats)); > + > + return 0; > +} > + > +/* > + * Port crypto Writer > + */ > +#ifdef RTE_PORT_STATS_COLLECT > + > +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) > \ > + port->stats.n_pkts_in += val > +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, > val) \ > + port->stats.n_pkts_drop += val > + > +#else > + > +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) > +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, > val) > + > +#endif > + > +struct rte_port_sym_crypto_writer { > + struct rte_port_out_stats stats; > + > + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; > + > + uint32_t tx_burst_sz; > + uint32_t tx_buf_count; > + uint64_t bsz_mask; > + > + uint8_t cryptodev_id; > + uint16_t queue_id; > + uint16_t crypto_op_offset; > +}; > + > +static void * > +rte_port_sym_crypto_writer_create(void *params, int socket_id) > +{ > + struct rte_port_sym_crypto_writer_params *conf = > + params; > + struct rte_port_sym_crypto_writer *port; > + > + /* Check input parameters */ > + if ((conf == NULL) || > + (conf->tx_burst_sz == 0) || > + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || > + (!rte_is_power_of_2(conf->tx_burst_sz))) { > + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", > __func__); > + return NULL; > + } > + > + /* Memory allocation */ > + port = rte_zmalloc_socket("PORT", sizeof(*port), > + RTE_CACHE_LINE_SIZE, socket_id); > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", > __func__); > + return NULL; > + } > + > + /* Initialization */ > + port->tx_burst_sz = conf->tx_burst_sz; > + port->tx_buf_count = 0; > + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); > + > + port->cryptodev_id = conf->cryptodev_id; > + port->queue_id = conf->queue_id; > + port->crypto_op_offset = conf->crypto_op_offset; > + > + return port; > +} > + > +static inline void > +send_burst(struct rte_port_sym_crypto_writer *p) > +{ > + uint32_t nb_tx; > + > + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, > + p->tx_buf, p->tx_buf_count); > + > + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p- > >tx_buf_count - > + nb_tx); > + for (; nb_tx < p->tx_buf_count; nb_tx++) > + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); > + > + p->tx_buf_count = 0; > +} > + > +static int > +rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) > +{ > + struct rte_port_sym_crypto_writer *p = > + port; > + > + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) > + RTE_MBUF_METADATA_UINT8_PTR(pkt, p- > >crypto_op_offset); > + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); > + if (p->tx_buf_count >= p->tx_burst_sz) > + send_burst(p); > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_tx_bulk(void *port, > + struct rte_mbuf **pkts, > + uint64_t pkts_mask) > +{ > + struct rte_port_sym_crypto_writer *p = > + port; > + uint64_t bsz_mask = p->bsz_mask; > + uint32_t tx_buf_count = p->tx_buf_count; > + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | > + ((pkts_mask & bsz_mask) ^ > bsz_mask); > + > + if (expr == 0) { > + uint64_t n_pkts = __builtin_popcountll(pkts_mask); > + uint32_t i; > + > + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, > n_pkts); > + > + for (i = 0; i < n_pkts; i++) > + p->tx_buf[p->tx_buf_count++] = (struct > rte_crypto_op *) > + > RTE_MBUF_METADATA_UINT8_PTR(pkts[i], > + p- > >crypto_op_offset); > + > + if (p->tx_buf_count >= p->tx_burst_sz) > + send_burst(p); > + } else { > + for (; pkts_mask;) { > + uint32_t pkt_index = __builtin_ctzll(pkts_mask); > + uint64_t pkt_mask = 1LLU << pkt_index; > + struct rte_mbuf *pkt = pkts[pkt_index]; > + > + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) > + > RTE_MBUF_METADATA_UINT8_PTR(pkt, > + p- > >crypto_op_offset); > + > + > RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); > + pkts_mask &= ~pkt_mask; > + } > + > + p->tx_buf_count = tx_buf_count; > + if (tx_buf_count >= p->tx_burst_sz) > + send_burst(p); > + } > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_flush(void *port) > +{ > + struct rte_port_sym_crypto_writer *p = > + port; > + > + if (p->tx_buf_count > 0) > + send_burst(p); > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_free(void *port) > +{ > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); > + return -EINVAL; > + } > + > + rte_port_sym_crypto_writer_flush(port); > + rte_free(port); > + > + return 0; > +} > + > +static int rte_port_sym_crypto_writer_stats_read(void *port, > + struct rte_port_out_stats *stats, int clear) > +{ > + struct rte_port_sym_crypto_writer *p = > + port; > + > + if (stats != NULL) > + memcpy(stats, &p->stats, sizeof(p->stats)); > + > + if (clear) > + memset(&p->stats, 0, sizeof(p->stats)); > + > + return 0; > +} > + > +/* > + * Port crypto Writer Nodrop > + */ > +#ifdef RTE_PORT_STATS_COLLECT > + > +#define > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, > val) \ > + port->stats.n_pkts_in += val > +#define > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, > val) \ > + port->stats.n_pkts_drop += val > + > +#else > + > +#define > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, > val) > +#define > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, > val) > + > +#endif > + > +struct rte_port_sym_crypto_writer_nodrop { > + struct rte_port_out_stats stats; > + > + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; > + uint32_t tx_burst_sz; > + uint32_t tx_buf_count; > + uint64_t bsz_mask; > + uint64_t n_retries; > + > + uint8_t cryptodev_id; > + uint16_t queue_id; > + uint16_t crypto_op_offset; > +}; > + > +static void * > +rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) > +{ > + struct rte_port_sym_crypto_writer_nodrop_params *conf = > + params; > + struct rte_port_sym_crypto_writer_nodrop *port; > + > + /* Check input parameters */ > + if ((conf == NULL) || > + (conf->tx_burst_sz == 0) || > + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || > + (!rte_is_power_of_2(conf->tx_burst_sz))) { > + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", > __func__); > + return NULL; > + } > + > + /* Memory allocation */ > + port = rte_zmalloc_socket("PORT", sizeof(*port), > + RTE_CACHE_LINE_SIZE, socket_id); > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", > __func__); > + return NULL; > + } > + > + /* Initialization */ > + port->cryptodev_id = conf->cryptodev_id; > + port->queue_id = conf->queue_id; > + port->crypto_op_offset = conf->crypto_op_offset; > + port->tx_burst_sz = conf->tx_burst_sz; > + port->tx_buf_count = 0; > + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); > + > + /* > + * When n_retries is 0 it means that we should wait for every packet > to > + * send no matter how many retries should it take. To limit number > of > + * branches in fast path, we use UINT64_MAX instead of branching. > + */ > + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf- > >n_retries; > + > + return port; > +} > + > +static inline void > +send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) > +{ > + uint32_t nb_tx = 0, i; > + > + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, > + p->tx_buf, p->tx_buf_count); > + > + /* We sent all the packets in a first try */ > + if (nb_tx >= p->tx_buf_count) { > + p->tx_buf_count = 0; > + return; > + } > + > + for (i = 0; i < p->n_retries; i++) { > + nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, > + p->queue_id, p->tx_buf + nb_tx, > + p->tx_buf_count - nb_tx); > + > + /* We sent all the packets in more than one try */ > + if (nb_tx >= p->tx_buf_count) { > + p->tx_buf_count = 0; > + return; > + } > + } > + > + /* We didn't send the packets in maximum allowed attempts */ > + > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_A > DD(p, > + p->tx_buf_count - nb_tx); > + for ( ; nb_tx < p->tx_buf_count; nb_tx++) > + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); > + > + p->tx_buf_count = 0; > +} > + > +static int > +rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) > +{ > + struct rte_port_sym_crypto_writer_nodrop *p = > + port; > + > + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) > + RTE_MBUF_METADATA_UINT8_PTR(pkt, p- > >crypto_op_offset); > + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); > + if (p->tx_buf_count >= p->tx_burst_sz) > + send_burst_nodrop(p); > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, > + struct rte_mbuf **pkts, > + uint64_t pkts_mask) > +{ > + struct rte_port_sym_crypto_writer_nodrop *p = > + port; > + > + uint64_t bsz_mask = p->bsz_mask; > + uint32_t tx_buf_count = p->tx_buf_count; > + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | > + ((pkts_mask & bsz_mask) ^ > bsz_mask); > + > + if (expr == 0) { > + uint64_t n_pkts = __builtin_popcountll(pkts_mask); > + uint32_t i; > + > + > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD( > p, n_pkts); > + > + for (i = 0; i < n_pkts; i++) > + p->tx_buf[p->tx_buf_count++] = (struct > rte_crypto_op *) > + > RTE_MBUF_METADATA_UINT8_PTR(pkts[i], > + p- > >crypto_op_offset); > + > + if (p->tx_buf_count >= p->tx_burst_sz) > + send_burst_nodrop(p); > + } else { > + for ( ; pkts_mask; ) { > + uint32_t pkt_index = __builtin_ctzll(pkts_mask); > + uint64_t pkt_mask = 1LLU << pkt_index; > + struct rte_mbuf *pkt = pkts[pkt_index]; > + > + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) > + > RTE_MBUF_METADATA_UINT8_PTR(pkt, > + p- > >crypto_op_offset); > + > RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD( > p, > + 1); > + pkts_mask &= ~pkt_mask; > + } > + > + p->tx_buf_count = tx_buf_count; > + if (tx_buf_count >= p->tx_burst_sz) > + send_burst_nodrop(p); > + } > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_nodrop_flush(void *port) > +{ > + struct rte_port_sym_crypto_writer_nodrop *p = > + port; > + > + if (p->tx_buf_count > 0) > + send_burst_nodrop(p); > + > + return 0; > +} > + > +static int > +rte_port_sym_crypto_writer_nodrop_free(void *port) > +{ > + if (port == NULL) { > + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); > + return -EINVAL; > + } > + > + rte_port_sym_crypto_writer_nodrop_flush(port); > + rte_free(port); > + > + return 0; > +} > + > +static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, > + struct rte_port_out_stats *stats, int clear) > +{ > + struct rte_port_sym_crypto_writer_nodrop *p = > + port; > + > + if (stats != NULL) > + memcpy(stats, &p->stats, sizeof(p->stats)); > + > + if (clear) > + memset(&p->stats, 0, sizeof(p->stats)); > + > + return 0; > +} > + > + > +/* > + * Summary of port operations > + */ > +struct rte_port_in_ops __rte_experimental > rte_port_sym_crypto_reader_ops = { > + .f_create = rte_port_sym_crypto_reader_create, > + .f_free = rte_port_sym_crypto_reader_free, > + .f_rx = rte_port_sym_crypto_reader_rx, > + .f_stats = rte_port_sym_crypto_reader_stats_read, > +}; > + > +struct rte_port_out_ops __rte_experimental > rte_port_sym_crypto_writer_ops = { > + .f_create = rte_port_sym_crypto_writer_create, > + .f_free = rte_port_sym_crypto_writer_free, > + .f_tx = rte_port_sym_crypto_writer_tx, > + .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, > + .f_flush = rte_port_sym_crypto_writer_flush, > + .f_stats = rte_port_sym_crypto_writer_stats_read, > +}; > + > +struct rte_port_out_ops __rte_experimental > +rte_port_sym_crypto_writer_nodrop_ops = { > + .f_create = rte_port_sym_crypto_writer_nodrop_create, > + .f_free = rte_port_sym_crypto_writer_nodrop_free, > + .f_tx = rte_port_sym_crypto_writer_nodrop_tx, > + .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, > + .f_flush = rte_port_sym_crypto_writer_nodrop_flush, > + .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, > +}; > diff --git a/lib/librte_port/rte_port_sym_crypto.h > b/lib/librte_port/rte_port_sym_crypto.h > new file mode 100644 > index 000000000..daf4a36dd > --- /dev/null > +++ b/lib/librte_port/rte_port_sym_crypto.h > @@ -0,0 +1,97 @@ > +/* SPDX-License-Identifier: BSD-3-Clause > + * Copyright(c) 2018 Intel Corporation > + */ > + > +#ifndef __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ > +#define __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ > + > +#ifdef __cplusplus > +extern "C" { > +#endif > + > +/** > + * @file > + * RTE Port sym crypto Interface > + * > + * crypto_reader: input port built on top of pre-initialized crypto interface > + * crypto_writer: output port built on top of pre-initialized crypto interface > + * > + * @warning > + * @b EXPERIMENTAL: this API may change without prior notice > + ***/ > + > +#include <stdint.h> > + > +#include <rte_cryptodev.h> > + > +#include "rte_port.h" > + > +/** Function prototype for reader post action. */ > +typedef void (*rte_port_sym_crypto_reader_callback_fn)(struct rte_mbuf > **pkts, > + uint16_t n_pkts, void *arg); > + > +/** Crypto_reader port parameters */ > +struct rte_port_sym_crypto_reader_params { > + /** Target cryptodev ID. */ > + uint8_t cryptodev_id; > + > + /** Target cryptodev Queue Pair ID. */ > + uint16_t queue_id; > + > + /** Crypto reader post callback function. */ > + rte_port_sym_crypto_reader_callback_fn f_callback; > + > + /** Crypto reader post callback function arguments. */ > + void *arg_callback; > +}; > + > +/** Crypto_reader port operations. */ > +extern struct rte_port_in_ops __rte_experimental > +rte_port_sym_crypto_reader_ops; > + > + > +/** Crypto_writer port parameters. */ > +struct rte_port_sym_crypto_writer_params { > + /** Target cryptodev ID. */ > + uint8_t cryptodev_id; > + > + /** Target cryptodev Queue Pair ID. */ > + uint16_t queue_id; > + > + /** offset to rte_crypto_op in the mbufs. */ > + uint16_t crypto_op_offset; > + > + /** Burst size to crypto interface. */ > + uint32_t tx_burst_sz; > +}; > + > +/** Crypto_writer port operations. */ > +extern struct rte_port_out_ops __rte_experimental > +rte_port_sym_crypto_writer_ops; > + > +/** Crypto_writer_nodrop port parameters. */ > +struct rte_port_sym_crypto_writer_nodrop_params { > + /** Target cryptodev ID. */ > + uint8_t cryptodev_id; > + > + /** Target cryptodev queue pair id. */ > + uint16_t queue_id; > + > + /** Offset to rte_crypto_op in the mbufs. */ > + uint16_t crypto_op_offset; > + > + /** Burst size to crypto interface. */ > + uint32_t tx_burst_sz; > + > + /** Maximum number of retries, 0 for no limit. */ > + uint32_t n_retries; > +}; > + > +/** Crypto_writer_nodrop port operations. */ > +extern struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops; > + > +#ifdef __cplusplus > +} > +#endif > + > +#endif > diff --git a/lib/librte_port/rte_port_version.map > b/lib/librte_port/rte_port_version.map > index 6470629b4..20e45a0ef 100644 > --- a/lib/librte_port/rte_port_version.map > +++ b/lib/librte_port/rte_port_version.map > @@ -51,3 +51,11 @@ DPDK_16.11 { > rte_port_fd_writer_nodrop_ops; > > } DPDK_16.07; > + > +EXPERIMENTAL { > + global: > + > + rte_port_sym_crypto_reader_ops; > + rte_port_sym_crypto_writer_ops; > + rte_port_sym_crypto_writer_nodrop_ops; > +}; > -- > 2.13.6 For some reason, not sure exactly why, there is this assembler warning, any idea of the cause? == Build lib/librte_port CC rte_port_sym_crypto.o /tmp/ccn9NRFm.s: Assembler messages: /tmp/ccn9NRFm.s:4984: Warning: setting incorrect section attributes for .text.experimental AR librte_port.a INSTALL-LIB librte_port.a ^ permalink raw reply [flat|nested] 6+ messages in thread
* [dpdk-dev] [PATCH v4] port: add sym crypto port 2018-09-28 10:41 ` [dpdk-dev] [PATCH v3] " Fan Zhang 2018-09-28 11:32 ` Dumitrescu, Cristian @ 2018-10-01 10:02 ` Fan Zhang 2018-10-01 17:25 ` Dumitrescu, Cristian 1 sibling, 1 reply; 6+ messages in thread From: Fan Zhang @ 2018-10-01 10:02 UTC (permalink / raw) To: dev; +Cc: cristian.dumitrescu, Zhang, Roy Fan, Zhang From: "Zhang, Roy Fan" <roy.fan.zhang@intel.com> This patch adds the symmetric crypto support to port library. The crypto port acts as a shim layer to DPDK cryptodev library and supports in-place crypto workload processing. Signed-off-by: Zhang, Roy Fan <roy.fan.zhang@intel.com> Acked-by: Dumitrescu, Cristian <cristian.dumitrescu@intel.com> --- v4: - fixed a compiler warning v3: - fixed a compile issue v2: - added experimental flags - enabled meson build lib/librte_port/Makefile | 4 +- lib/librte_port/meson.build | 8 +- lib/librte_port/rte_port_sym_crypto.c | 552 ++++++++++++++++++++++++++++++++++ lib/librte_port/rte_port_sym_crypto.h | 93 ++++++ lib/librte_port/rte_port_version.map | 9 + 5 files changed, 662 insertions(+), 4 deletions(-) create mode 100644 lib/librte_port/rte_port_sym_crypto.c create mode 100644 lib/librte_port/rte_port_sym_crypto.h diff --git a/lib/librte_port/Makefile b/lib/librte_port/Makefile index 8df4864ed..1b83f6f2b 100644 --- a/lib/librte_port/Makefile +++ b/lib/librte_port/Makefile @@ -11,7 +11,7 @@ ifeq ($(CONFIG_RTE_PORT_PCAP),y) LDLIBS += -lpcap endif LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev -LDLIBS += -lrte_ip_frag -lrte_sched +LDLIBS += -lrte_ip_frag -lrte_sched -lrte_cryptodev ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) LDLIBS += -lrte_kni endif @@ -38,6 +38,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_kni.c endif SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_source_sink.c +SRCS-$(CONFIG_RTE_LIBRTE_PORT) += rte_port_sym_crypto.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port.h @@ -53,5 +54,6 @@ ifeq ($(CONFIG_RTE_LIBRTE_KNI),y) SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_kni.h endif SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_source_sink.h +SYMLINK-$(CONFIG_RTE_LIBRTE_PORT)-include += rte_port_sym_crypto.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_port/meson.build b/lib/librte_port/meson.build index f3d8b4434..0d11456f0 100644 --- a/lib/librte_port/meson.build +++ b/lib/librte_port/meson.build @@ -9,7 +9,8 @@ sources = files( 'rte_port_ras.c', 'rte_port_ring.c', 'rte_port_sched.c', - 'rte_port_source_sink.c') + 'rte_port_source_sink.c', + 'rte_port_sym_crypto.c') headers = files( 'rte_port_ethdev.h', 'rte_port_fd.h', @@ -18,8 +19,9 @@ headers = files( 'rte_port.h', 'rte_port_ring.h', 'rte_port_sched.h', - 'rte_port_source_sink.h') -deps += ['ethdev', 'sched', 'ip_frag'] + 'rte_port_source_sink.h', + 'rte_port_sym_crypto.h') +deps += ['ethdev', 'sched', 'ip_frag', 'cryptodev'] if dpdk_conf.has('RTE_LIBRTE_KNI') sources += files('rte_port_kni.c') diff --git a/lib/librte_port/rte_port_sym_crypto.c b/lib/librte_port/rte_port_sym_crypto.c new file mode 100644 index 000000000..1cd242e22 --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.c @@ -0,0 +1,552 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ +#include <string.h> + +#include <rte_common.h> +#include <rte_malloc.h> + +#include "rte_port_sym_crypto.h" + +/* + * Port Crypto Reader + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_reader { + struct rte_port_in_stats stats; + + uint8_t cryptodev_id; + uint16_t queue_id; + struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX]; + rte_port_sym_crypto_reader_callback_fn f_callback; + void *arg_callback; +}; + +static void * +rte_port_sym_crypto_reader_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_reader_params *conf = + params; + struct rte_port_sym_crypto_reader *port; + + /* Check input parameters */ + if (conf == NULL) { + RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->f_callback = conf->f_callback; + port->arg_callback = conf->arg_callback; + + return port; +} + +static int +rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts) +{ + struct rte_port_sym_crypto_reader *p = + port; + uint16_t rx_ops_cnt, i, n = 0; + + rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id, + p->ops, n_pkts); + + for (i = 0; i < rx_ops_cnt; i++) { + struct rte_crypto_op *op = p->ops[i]; + + /** Drop failed pkts */ + if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) { + rte_pktmbuf_free(op->sym->m_src); + continue; + } + + pkts[n++] = op->sym->m_src; + } + + if (p->f_callback) + (*p->f_callback)(pkts, n, p->arg_callback); + + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n); + RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_pkt_cnt - n); + + return n; +} + +static int +rte_port_sym_crypto_reader_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__); + return -EINVAL; + } + + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_reader_stats_read(void *port, + struct rte_port_in_stats *stats, int clear) +{ + struct rte_port_sym_crypto_reader *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_params *conf = + params; + struct rte_port_sym_crypto_writer *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + + return port; +} + +static inline void +send_burst(struct rte_port_sym_crypto_writer *p) +{ + uint32_t nb_tx; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - + nb_tx); + for (; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer *p = + port; + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } else { + for (; pkts_mask;) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_flush(void *port) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (p->tx_buf_count > 0) + send_burst(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + +/* + * Port crypto Writer Nodrop + */ +#ifdef RTE_PORT_STATS_COLLECT + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \ + port->stats.n_pkts_in += val +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \ + port->stats.n_pkts_drop += val + +#else + +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) +#define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) + +#endif + +struct rte_port_sym_crypto_writer_nodrop { + struct rte_port_out_stats stats; + + struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX]; + uint32_t tx_burst_sz; + uint32_t tx_buf_count; + uint64_t bsz_mask; + uint64_t n_retries; + + uint8_t cryptodev_id; + uint16_t queue_id; + uint16_t crypto_op_offset; +}; + +static void * +rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id) +{ + struct rte_port_sym_crypto_writer_nodrop_params *conf = + params; + struct rte_port_sym_crypto_writer_nodrop *port; + + /* Check input parameters */ + if ((conf == NULL) || + (conf->tx_burst_sz == 0) || + (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) || + (!rte_is_power_of_2(conf->tx_burst_sz))) { + RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__); + return NULL; + } + + /* Memory allocation */ + port = rte_zmalloc_socket("PORT", sizeof(*port), + RTE_CACHE_LINE_SIZE, socket_id); + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__); + return NULL; + } + + /* Initialization */ + port->cryptodev_id = conf->cryptodev_id; + port->queue_id = conf->queue_id; + port->crypto_op_offset = conf->crypto_op_offset; + port->tx_burst_sz = conf->tx_burst_sz; + port->tx_buf_count = 0; + port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1); + + /* + * When n_retries is 0 it means that we should wait for every packet to + * send no matter how many retries should it take. To limit number of + * branches in fast path, we use UINT64_MAX instead of branching. + */ + port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries; + + return port; +} + +static inline void +send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p) +{ + uint32_t nb_tx = 0, i; + + nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id, + p->tx_buf, p->tx_buf_count); + + /* We sent all the packets in a first try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + + for (i = 0; i < p->n_retries; i++) { + nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id, + p->queue_id, p->tx_buf + nb_tx, + p->tx_buf_count - nb_tx); + + /* We sent all the packets in more than one try */ + if (nb_tx >= p->tx_buf_count) { + p->tx_buf_count = 0; + return; + } + } + + /* We didn't send the packets in maximum allowed attempts */ + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p, + p->tx_buf_count - nb_tx); + for ( ; nb_tx < p->tx_buf_count; nb_tx++) + rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src); + + p->tx_buf_count = 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1); + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port, + struct rte_mbuf **pkts, + uint64_t pkts_mask) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + uint64_t bsz_mask = p->bsz_mask; + uint32_t tx_buf_count = p->tx_buf_count; + uint64_t expr = (pkts_mask & (pkts_mask + 1)) | + ((pkts_mask & bsz_mask) ^ bsz_mask); + + if (expr == 0) { + uint64_t n_pkts = __builtin_popcountll(pkts_mask); + uint32_t i; + + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); + + for (i = 0; i < n_pkts; i++) + p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkts[i], + p->crypto_op_offset); + + if (p->tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } else { + for ( ; pkts_mask; ) { + uint32_t pkt_index = __builtin_ctzll(pkts_mask); + uint64_t pkt_mask = 1LLU << pkt_index; + struct rte_mbuf *pkt = pkts[pkt_index]; + + p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *) + RTE_MBUF_METADATA_UINT8_PTR(pkt, + p->crypto_op_offset); + RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, + 1); + pkts_mask &= ~pkt_mask; + } + + p->tx_buf_count = tx_buf_count; + if (tx_buf_count >= p->tx_burst_sz) + send_burst_nodrop(p); + } + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_flush(void *port) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (p->tx_buf_count > 0) + send_burst_nodrop(p); + + return 0; +} + +static int +rte_port_sym_crypto_writer_nodrop_free(void *port) +{ + if (port == NULL) { + RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__); + return -EINVAL; + } + + rte_port_sym_crypto_writer_nodrop_flush(port); + rte_free(port); + + return 0; +} + +static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port, + struct rte_port_out_stats *stats, int clear) +{ + struct rte_port_sym_crypto_writer_nodrop *p = + port; + + if (stats != NULL) + memcpy(stats, &p->stats, sizeof(p->stats)); + + if (clear) + memset(&p->stats, 0, sizeof(p->stats)); + + return 0; +} + + +/* + * Summary of port operations + */ +struct rte_port_in_ops rte_port_sym_crypto_reader_ops = { + .f_create = rte_port_sym_crypto_reader_create, + .f_free = rte_port_sym_crypto_reader_free, + .f_rx = rte_port_sym_crypto_reader_rx, + .f_stats = rte_port_sym_crypto_reader_stats_read, +}; + +struct rte_port_out_ops rte_port_sym_crypto_writer_ops = { + .f_create = rte_port_sym_crypto_writer_create, + .f_free = rte_port_sym_crypto_writer_free, + .f_tx = rte_port_sym_crypto_writer_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_flush, + .f_stats = rte_port_sym_crypto_writer_stats_read, +}; + +struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = { + .f_create = rte_port_sym_crypto_writer_nodrop_create, + .f_free = rte_port_sym_crypto_writer_nodrop_free, + .f_tx = rte_port_sym_crypto_writer_nodrop_tx, + .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk, + .f_flush = rte_port_sym_crypto_writer_nodrop_flush, + .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read, +}; diff --git a/lib/librte_port/rte_port_sym_crypto.h b/lib/librte_port/rte_port_sym_crypto.h new file mode 100644 index 000000000..181f6ce01 --- /dev/null +++ b/lib/librte_port/rte_port_sym_crypto.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ +#define __INCLUDE_RTE_PORT_SYM_CRYPTO_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @file + * RTE Port sym crypto Interface + * + * crypto_reader: input port built on top of pre-initialized crypto interface + * crypto_writer: output port built on top of pre-initialized crypto interface + * + **/ + +#include <stdint.h> + +#include <rte_cryptodev.h> + +#include "rte_port.h" + +/** Function prototype for reader post action. */ +typedef void (*rte_port_sym_crypto_reader_callback_fn)(struct rte_mbuf **pkts, + uint16_t n_pkts, void *arg); + +/** Crypto_reader port parameters */ +struct rte_port_sym_crypto_reader_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** Crypto reader post callback function. */ + rte_port_sym_crypto_reader_callback_fn f_callback; + + /** Crypto reader post callback function arguments. */ + void *arg_callback; +}; + +/** Crypto_reader port operations. */ +extern struct rte_port_in_ops rte_port_sym_crypto_reader_ops; + + +/** Crypto_writer port parameters. */ +struct rte_port_sym_crypto_writer_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev Queue Pair ID. */ + uint16_t queue_id; + + /** offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; +}; + +/** Crypto_writer port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_ops; + +/** Crypto_writer_nodrop port parameters. */ +struct rte_port_sym_crypto_writer_nodrop_params { + /** Target cryptodev ID. */ + uint8_t cryptodev_id; + + /** Target cryptodev queue pair id. */ + uint16_t queue_id; + + /** Offset to rte_crypto_op in the mbufs. */ + uint16_t crypto_op_offset; + + /** Burst size to crypto interface. */ + uint32_t tx_burst_sz; + + /** Maximum number of retries, 0 for no limit. */ + uint32_t n_retries; +}; + +/** Crypto_writer_nodrop port operations. */ +extern struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/lib/librte_port/rte_port_version.map b/lib/librte_port/rte_port_version.map index 6470629b4..609bcec3f 100644 --- a/lib/librte_port/rte_port_version.map +++ b/lib/librte_port/rte_port_version.map @@ -51,3 +51,12 @@ DPDK_16.11 { rte_port_fd_writer_nodrop_ops; } DPDK_16.07; + +DPDK_18.11 { + global: + + rte_port_sym_crypto_reader_ops; + rte_port_sym_crypto_writer_ops; + rte_port_sym_crypto_writer_nodrop_ops; + +} DPDK_16.11; -- 2.13.6 ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [dpdk-dev] [PATCH v4] port: add sym crypto port 2018-10-01 10:02 ` [dpdk-dev] [PATCH v4] " Fan Zhang @ 2018-10-01 17:25 ` Dumitrescu, Cristian 0 siblings, 0 replies; 6+ messages in thread From: Dumitrescu, Cristian @ 2018-10-01 17:25 UTC (permalink / raw) To: Zhang, Roy Fan, dev > -----Original Message----- > From: Zhang, Roy Fan > Sent: Monday, October 1, 2018 11:02 AM > To: dev@dpdk.org > Cc: Dumitrescu, Cristian <cristian.dumitrescu@intel.com>; Zhang, Roy Fan > <roy.fan.zhang@intel.com>; Zhang > Subject: [PATCH v4] port: add sym crypto port > > From: "Zhang, Roy Fan" <roy.fan.zhang@intel.com> > > This patch adds the symmetric crypto support to port library. > The crypto port acts as a shim layer to DPDK cryptodev library and > supports in-place crypto workload processing. > > Signed-off-by: Zhang, Roy Fan <roy.fan.zhang@intel.com> > Acked-by: Dumitrescu, Cristian <cristian.dumitrescu@intel.com> > --- > v4: > - fixed a compiler warning > > v3: > - fixed a compile issue > > v2: > - added experimental flags > - enabled meson build > > lib/librte_port/Makefile | 4 +- > lib/librte_port/meson.build | 8 +- > lib/librte_port/rte_port_sym_crypto.c | 552 > ++++++++++++++++++++++++++++++++++ Fixed some minor style errors. Applied to next-pipeline tree, thanks! ^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2018-10-01 17:25 UTC | newest] Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2018-08-20 11:38 [dpdk-dev] [PATCH] port: add sym crypto port Zhang, Roy Fan 2018-08-20 14:29 ` [dpdk-dev] [PATCH v2] " Zhang, Roy Fan 2018-09-28 10:41 ` [dpdk-dev] [PATCH v3] " Fan Zhang 2018-09-28 11:32 ` Dumitrescu, Cristian 2018-10-01 10:02 ` [dpdk-dev] [PATCH v4] " Fan Zhang 2018-10-01 17:25 ` Dumitrescu, Cristian
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).