From: Michal Krawczyk <mk@semihalf.com>
To: dev@dpdk.org
Cc: gtzalik@dpdk.org, mw@dpdk.org, matua@amazon.com, rk@semihalf.com,
Michal Krawczyk <mk@semihalf.com>
Subject: [dpdk-dev] [PATCH 06/20] net/ena: add LLQv2 support
Date: Fri, 14 Dec 2018 14:18:32 +0100 [thread overview]
Message-ID: <20181214131846.22439-7-mk@semihalf.com> (raw)
In-Reply-To: <20181214131846.22439-1-mk@semihalf.com>
LLQ (Low Latency Queue) is the feature that allows pushing header
directly to the device through PCI before even DMA is triggered.
It reduces latency, bacause device can start preparing packet before
payload is sent through DMA.
Signed-off-by: Michal Krawczyk <mk@semihalf.com>
---
drivers/net/ena/ena_ethdev.c | 220 ++++++++++++++++++++++++++++++++++++-------
drivers/net/ena/ena_ethdev.h | 3 +
2 files changed, 190 insertions(+), 33 deletions(-)
diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c
index 505d9bf30..73070ad96 100644
--- a/drivers/net/ena/ena_ethdev.c
+++ b/drivers/net/ena/ena_ethdev.c
@@ -116,6 +116,9 @@ struct ena_stats {
#define ENA_STAT_GLOBAL_ENTRY(stat) \
ENA_STAT_ENTRY(stat, dev)
+#define ENA_MAX_RING_SIZE_RX 1024
+#define ENA_MAX_RING_SIZE_TX 1024
+
/*
* Each rte_memzone should have unique name.
* To satisfy it, count number of allocation and add it to name.
@@ -806,6 +809,9 @@ static void ena_tx_queue_release(void *queue)
ena_tx_queue_release_bufs(ring);
/* Free ring resources */
+ if (ring->push_buf_intermediate_buf)
+ rte_free(ring->push_buf_intermediate_buf);
+
if (ring->tx_buffer_info)
rte_free(ring->tx_buffer_info);
@@ -814,6 +820,7 @@ static void ena_tx_queue_release(void *queue)
ring->empty_tx_reqs = NULL;
ring->tx_buffer_info = NULL;
+ ring->push_buf_intermediate_buf = NULL;
ring->configured = 0;
@@ -937,15 +944,30 @@ static int ena_check_valid_conf(struct ena_adapter *adapter)
static int
ena_calc_queue_size(struct ena_calc_queue_size_ctx *ctx)
{
- uint32_t tx_queue_size, rx_queue_size;
+ struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
+ struct ena_com_dev *ena_dev = ctx->ena_dev;
+ uint32_t tx_queue_size = ENA_MAX_RING_SIZE_TX;
+ uint32_t rx_queue_size = ENA_MAX_RING_SIZE_RX;
- if (ctx->ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
+ if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
struct ena_admin_queue_ext_feature_fields *max_queue_ext =
&ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
- rx_queue_size = RTE_MIN(max_queue_ext->max_rx_cq_depth,
+ rx_queue_size = RTE_MIN(rx_queue_size,
+ max_queue_ext->max_rx_cq_depth);
+ rx_queue_size = RTE_MIN(rx_queue_size,
max_queue_ext->max_rx_sq_depth);
- tx_queue_size = RTE_MIN(max_queue_ext->max_tx_cq_depth,
- max_queue_ext->max_tx_sq_depth);
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ max_queue_ext->max_tx_cq_depth);
+
+ if (ena_dev->tx_mem_queue_type ==
+ ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ llq->max_llq_depth);
+ } else {
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ max_queue_ext->max_tx_sq_depth);
+ }
+
ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
max_queue_ext->max_per_packet_rx_descs);
ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
@@ -953,9 +975,22 @@ ena_calc_queue_size(struct ena_calc_queue_size_ctx *ctx)
} else {
struct ena_admin_queue_feature_desc *max_queues =
&ctx->get_feat_ctx->max_queues;
- rx_queue_size = RTE_MIN(max_queues->max_cq_depth,
+ rx_queue_size = RTE_MIN(rx_queue_size,
+ max_queues->max_cq_depth);
+ rx_queue_size = RTE_MIN(rx_queue_size,
max_queues->max_sq_depth);
- tx_queue_size = rx_queue_size;
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ max_queues->max_cq_depth);
+
+ if (ena_dev->tx_mem_queue_type ==
+ ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ llq->max_llq_depth);
+ } else {
+ tx_queue_size = RTE_MIN(tx_queue_size,
+ max_queues->max_sq_depth);
+ }
+
ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
max_queues->max_packet_tx_descs);
ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
@@ -1277,6 +1312,17 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,
return -ENOMEM;
}
+ txq->push_buf_intermediate_buf =
+ rte_zmalloc("txq->push_buf_intermediate_buf",
+ txq->tx_max_header_size,
+ RTE_CACHE_LINE_SIZE);
+ if (!txq->push_buf_intermediate_buf) {
+ RTE_LOG(ERR, PMD, "failed to alloc push buff for LLQ\n");
+ rte_free(txq->tx_buffer_info);
+ rte_free(txq->empty_tx_reqs);
+ return -ENOMEM;
+ }
+
for (i = 0; i < txq->ring_size; i++)
txq->empty_tx_reqs[i] = i;
@@ -1592,28 +1638,87 @@ static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer,
}
}
+static inline void
+set_default_llq_configurations(struct ena_llq_configurations *llq_config)
+{
+ llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
+ llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
+ llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
+ llq_config->llq_num_decs_before_header =
+ ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
+ llq_config->llq_ring_entry_size_value = 128;
+}
+
+static int
+ena_set_queues_placement_policy(struct ena_adapter *adapter,
+ struct ena_com_dev *ena_dev,
+ struct ena_admin_feature_llq_desc *llq,
+ struct ena_llq_configurations *llq_default_configurations)
+{
+ int rc;
+ u32 llq_feature_mask;
+
+ llq_feature_mask = 1 << ENA_ADMIN_LLQ;
+ if (!(ena_dev->supported_features & llq_feature_mask)) {
+ RTE_LOG(INFO, PMD,
+ "LLQ is not supported. Fallback to host mode policy.\n");
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+ return 0;
+ }
+
+ rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
+ if (unlikely(rc)) {
+ PMD_INIT_LOG(WARNING, "Failed to config dev mode. "
+ "Fallback to host mode policy.\n");
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+ return 0;
+ }
+
+ /* Nothing to config, exit */
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+ return 0;
+
+ if (!adapter->dev_mem_base) {
+ RTE_LOG(ERR, PMD, "Unable to access LLQ bar resource. "
+ "Fallback to host mode policy.\n.");
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+ return 0;
+ }
+
+ ena_dev->mem_bar = adapter->dev_mem_base;
+
+ return 0;
+}
+
static int ena_calc_io_queue_num(struct ena_com_dev *ena_dev,
struct ena_com_dev_get_features_ctx *get_feat_ctx)
{
- uint32_t io_sq_num, io_cq_num, io_queue_num;
+ uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, io_queue_num;
/* Regular queues capabilities */
if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
struct ena_admin_queue_ext_feature_fields *max_queue_ext =
&get_feat_ctx->max_queue_ext.max_queue_ext;
- io_sq_num = max_queue_ext->max_rx_sq_num;
- io_sq_num = RTE_MIN(io_sq_num, max_queue_ext->max_tx_sq_num);
-
- io_cq_num = max_queue_ext->max_rx_cq_num;
- io_cq_num = RTE_MIN(io_cq_num, max_queue_ext->max_tx_cq_num);
+ io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num,
+ max_queue_ext->max_rx_cq_num);
+ io_tx_sq_num = max_queue_ext->max_tx_sq_num;
+ io_tx_cq_num = max_queue_ext->max_tx_cq_num;
} else {
struct ena_admin_queue_feature_desc *max_queues =
&get_feat_ctx->max_queues;
- io_sq_num = max_queues->max_sq_num;
- io_cq_num = max_queues->max_cq_num;
+ io_tx_sq_num = max_queues->max_sq_num;
+ io_tx_cq_num = max_queues->max_cq_num;
+ io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num);
}
- io_queue_num = RTE_MIN(io_sq_num, io_cq_num);
+ /* In case of LLQ use the llq number in the get feature cmd */
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
+ io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
+
+ io_queue_num = RTE_MIN(rte_lcore_count(), ENA_MAX_NUM_IO_QUEUES);
+ io_queue_num = RTE_MIN(io_queue_num, io_rx_num);
+ io_queue_num = RTE_MIN(io_queue_num, io_tx_sq_num);
+ io_queue_num = RTE_MIN(io_queue_num, io_tx_cq_num);
if (unlikely(io_queue_num == 0)) {
RTE_LOG(ERR, PMD, "Number of IO queues should not be 0\n");
@@ -1632,6 +1737,8 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
(struct ena_adapter *)(eth_dev->data->dev_private);
struct ena_com_dev *ena_dev = &adapter->ena_dev;
struct ena_com_dev_get_features_ctx get_feat_ctx;
+ struct ena_llq_configurations llq_config;
+ const char *queue_type_str;
int rc;
static int adapters_found;
@@ -1686,11 +1793,22 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
}
adapter->wd_state = wd_state;
- ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+ set_default_llq_configurations(&llq_config);
+ rc = ena_set_queues_placement_policy(adapter, ena_dev,
+ &get_feat_ctx.llq, &llq_config);
+ if (unlikely(rc)) {
+ PMD_INIT_LOG(CRIT, "Failed to set placement policy");
+ return rc;
+ }
+
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+ queue_type_str = "Regular";
+ else
+ queue_type_str = "Low latency";
+ RTE_LOG(INFO, PMD, "Placement policy: %s\n", queue_type_str);
calc_queue_ctx.ena_dev = ena_dev;
calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
-
adapter->num_queues = ena_calc_io_queue_num(ena_dev,
&get_feat_ctx);
@@ -2106,13 +2224,21 @@ static void ena_update_hints(struct ena_adapter *adapter,
static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring,
struct rte_mbuf *mbuf)
{
- int num_segments, rc;
+ struct ena_com_dev *ena_dev;
+ int num_segments, header_len, rc;
+ ena_dev = &tx_ring->adapter->ena_dev;
num_segments = mbuf->nb_segs;
+ header_len = mbuf->data_len;
if (likely(num_segments < tx_ring->sgl_size))
return 0;
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV &&
+ (num_segments == tx_ring->sgl_size) &&
+ (header_len < tx_ring->tx_max_header_size))
+ return 0;
+
rc = rte_pktmbuf_linearize(mbuf);
if (unlikely(rc))
RTE_LOG(WARNING, PMD, "Mbuf linearize failed\n");
@@ -2127,6 +2253,7 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t next_to_use = tx_ring->next_to_use;
uint16_t next_to_clean = tx_ring->next_to_clean;
struct rte_mbuf *mbuf;
+ uint16_t seg_len;
unsigned int ring_size = tx_ring->ring_size;
unsigned int ring_mask = ring_size - 1;
struct ena_com_tx_ctx ena_tx_ctx;
@@ -2134,6 +2261,8 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
struct ena_com_buf *ebuf;
uint16_t rc, req_id, total_tx_descs = 0;
uint16_t sent_idx = 0, empty_tx_reqs;
+ uint16_t push_len = 0;
+ uint16_t delta = 0;
int nb_hw_desc;
/* Check adapter state */
@@ -2166,17 +2295,32 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
sizeof(struct ena_com_tx_meta));
ena_tx_ctx.ena_bufs = ebuf;
ena_tx_ctx.req_id = req_id;
+
+ delta = 0;
+ seg_len = mbuf->data_len;
+
if (tx_ring->tx_mem_queue_type ==
ENA_ADMIN_PLACEMENT_POLICY_DEV) {
- /* prepare the push buffer with
- * virtual address of the data
- */
- ena_tx_ctx.header_len =
- RTE_MIN(mbuf->data_len,
- tx_ring->tx_max_header_size);
- ena_tx_ctx.push_header =
- (void *)((char *)mbuf->buf_addr +
- mbuf->data_off);
+ push_len = RTE_MIN(mbuf->pkt_len,
+ tx_ring->tx_max_header_size);
+ ena_tx_ctx.header_len = push_len;
+
+ if (likely(push_len <= seg_len)) {
+ /* If the push header is in the single segment,
+ * then just point it to the 1st mbuf data.
+ */
+ ena_tx_ctx.push_header =
+ rte_pktmbuf_mtod(mbuf, uint8_t *);
+ } else {
+ /* If the push header lays in the several
+ * segments, copy it to the intermediate buffer.
+ */
+ rte_pktmbuf_read(mbuf, 0, push_len,
+ tx_ring->push_buf_intermediate_buf);
+ ena_tx_ctx.push_header =
+ tx_ring->push_buf_intermediate_buf;
+ delta = push_len - seg_len;
+ }
} /* there's no else as we take advantage of memset zeroing */
/* Set TX offloads flags, if applicable */
@@ -2191,20 +2335,30 @@ static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
/* Process first segment taking into
* consideration pushed header
*/
- if (mbuf->data_len > ena_tx_ctx.header_len) {
+ if (seg_len > push_len) {
ebuf->paddr = mbuf->buf_iova +
mbuf->data_off +
- ena_tx_ctx.header_len;
- ebuf->len = mbuf->data_len - ena_tx_ctx.header_len;
+ push_len;
+ ebuf->len = seg_len - push_len;
ebuf++;
tx_info->num_of_bufs++;
}
while ((mbuf = mbuf->next) != NULL) {
- ebuf->paddr = mbuf->buf_iova + mbuf->data_off;
- ebuf->len = mbuf->data_len;
+ seg_len = mbuf->data_len;
+
+ /* Skip mbufs if whole data is pushed as a header */
+ if (unlikely(delta > seg_len)) {
+ delta -= seg_len;
+ continue;
+ }
+
+ ebuf->paddr = mbuf->buf_iova + mbuf->data_off + delta;
+ ebuf->len = seg_len - delta;
ebuf++;
tx_info->num_of_bufs++;
+
+ delta = 0;
}
ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
diff --git a/drivers/net/ena/ena_ethdev.h b/drivers/net/ena/ena_ethdev.h
index e6f7bd012..713cdea97 100644
--- a/drivers/net/ena/ena_ethdev.h
+++ b/drivers/net/ena/ena_ethdev.h
@@ -110,6 +110,9 @@ struct ena_ring {
/* Max length PMD can push to device for LLQ */
uint8_t tx_max_header_size;
int configured;
+
+ uint8_t *push_buf_intermediate_buf;
+
struct ena_adapter *adapter;
uint64_t offloads;
u16 sgl_size;
--
2.14.1
next prev parent reply other threads:[~2018-12-14 13:19 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-12-14 13:18 [dpdk-dev] [PATCH 00/20] net/ena: ENAv2 release Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 01/20] net/ena: update ena_com for the ENAv2 Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 02/20] net/ena: pass number of CPUs to the host info structure Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 03/20] net/ena: add reset reason in rx error Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 04/20] net/ena: add hw queues depth setup Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 05/20] net/ena: skip packet with wrong req_id Michal Krawczyk
2018-12-14 13:18 ` Michal Krawczyk [this message]
2018-12-14 13:18 ` [dpdk-dev] [PATCH 07/20] net/ena: increase maximum Rx ring size Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 08/20] net/ena: call additional doorbells if needed Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 09/20] net/ena: destroy queues if start failed Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 10/20] net/ena: reset should not reconfigure queues Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 11/20] net/ena: adjust new line in log messages Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 12/20] net/ena: add supported RSS offloads types Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 13/20] net/ena: fix invalid reference to variable in union Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 14/20] net/ena: fix cleanup for out of order packets Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 15/20] net/ena: add per-queue software counters stats Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 16/20] net/ena: expose ENA-specific statistics in extended stats Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 17/20] net/ena: new way of getting rx drops Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 18/20] net/ena: update completion queue after cleanup Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 19/20] docs: update ENA PMD documentation and features Michal Krawczyk
2018-12-14 13:18 ` [dpdk-dev] [PATCH 20/20] net/ena: release of the ENAv2 driver (v2.0.0) Michal Krawczyk
2018-12-17 11:03 ` [dpdk-dev] [PATCH v2 14/20] net/ena: fix cleanup for out of order packets Michal Krawczyk
2018-12-17 11:06 ` [dpdk-dev] [PATCH v3 " Michal Krawczyk
2018-12-18 17:18 ` [dpdk-dev] [PATCH 00/20] net/ena: ENAv2 release Ferruh Yigit
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20181214131846.22439-7-mk@semihalf.com \
--to=mk@semihalf.com \
--cc=dev@dpdk.org \
--cc=gtzalik@dpdk.org \
--cc=matua@amazon.com \
--cc=mw@dpdk.org \
--cc=rk@semihalf.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).