From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id EE2E7A0C43; Fri, 27 Aug 2021 08:58:27 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8E6D04126C; Fri, 27 Aug 2021 08:58:03 +0200 (CEST) Received: from shelob.oktetlabs.ru (shelob.oktetlabs.ru [91.220.146.113]) by mails.dpdk.org (Postfix) with ESMTP id F059040DF4 for ; Fri, 27 Aug 2021 08:58:01 +0200 (CEST) Received: by shelob.oktetlabs.ru (Postfix, from userid 122) id C7FD67F6FC; Fri, 27 Aug 2021 09:58:01 +0300 (MSK) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on shelob.oktetlabs.ru X-Spam-Level: X-Spam-Status: No, score=0.8 required=5.0 tests=ALL_TRUSTED, DKIM_ADSP_DISCARD, URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.2 Received: from aros.oktetlabs.ru (aros.oktetlabs.ru [192.168.38.17]) by shelob.oktetlabs.ru (Postfix) with ESMTP id A1D577F6D8; Fri, 27 Aug 2021 09:57:33 +0300 (MSK) DKIM-Filter: OpenDKIM Filter v2.11.0 shelob.oktetlabs.ru A1D577F6D8 Authentication-Results: shelob.oktetlabs.ru/A1D577F6D8; dkim=none; dkim-atps=neutral From: Andrew Rybchenko To: dev@dpdk.org Cc: Igor Romanov , Andy Moreton , Ivan Malov Date: Fri, 27 Aug 2021 09:56:47 +0300 Message-Id: <20210827065717.1838258-9-andrew.rybchenko@oktetlabs.ru> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210827065717.1838258-1-andrew.rybchenko@oktetlabs.ru> References: <20210827065717.1838258-1-andrew.rybchenko@oktetlabs.ru> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH 08/38] net/sfc: reserve TxQ and RxQ for port representors X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Igor Romanov A Tx/Rx queue pair is required to forward traffic between port representors and virtual functions. Signed-off-by: Igor Romanov Signed-off-by: Andrew Rybchenko Reviewed-by: Andy Moreton Reviewed-by: Ivan Malov --- drivers/net/sfc/sfc.c | 51 ++++++++++++++++++++++++++++++-- drivers/net/sfc/sfc.h | 15 ++++++++++ drivers/net/sfc/sfc_ev.h | 40 ++++++++++++++++++------- drivers/net/sfc/sfc_repr_proxy.c | 12 +++++--- drivers/net/sfc/sfc_repr_proxy.h | 8 +++++ drivers/net/sfc/sfc_tx.c | 29 ++++++++++-------- 6 files changed, 124 insertions(+), 31 deletions(-) diff --git a/drivers/net/sfc/sfc.c b/drivers/net/sfc/sfc.c index 591b8971b3..9abd6d600b 100644 --- a/drivers/net/sfc/sfc.c +++ b/drivers/net/sfc/sfc.c @@ -46,6 +46,12 @@ sfc_repr_supported(const struct sfc_adapter *sa) return true; } +bool +sfc_repr_available(const struct sfc_adapter_shared *sas) +{ + return sas->nb_repr_rxq > 0 && sas->nb_repr_txq > 0; +} + int sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, size_t len, int socket_id, efsys_mem_t *esmp) @@ -296,6 +302,41 @@ sfc_estimate_resource_limits(struct sfc_adapter *sa) sas->counters_rxq_allocated = false; } + if (sfc_repr_supported(sa) && + evq_allocated >= SFC_REPR_PROXY_NB_RXQ_MIN + + SFC_REPR_PROXY_NB_TXQ_MIN && + rxq_allocated >= SFC_REPR_PROXY_NB_RXQ_MIN && + txq_allocated >= SFC_REPR_PROXY_NB_TXQ_MIN) { + unsigned int extra; + + txq_allocated -= SFC_REPR_PROXY_NB_TXQ_MIN; + rxq_allocated -= SFC_REPR_PROXY_NB_RXQ_MIN; + evq_allocated -= SFC_REPR_PROXY_NB_RXQ_MIN + + SFC_REPR_PROXY_NB_TXQ_MIN; + + sas->nb_repr_rxq = SFC_REPR_PROXY_NB_RXQ_MIN; + sas->nb_repr_txq = SFC_REPR_PROXY_NB_TXQ_MIN; + + /* Allocate extra representor RxQs up to the maximum */ + extra = MIN(evq_allocated, rxq_allocated); + extra = MIN(extra, + SFC_REPR_PROXY_NB_RXQ_MAX - sas->nb_repr_rxq); + evq_allocated -= extra; + rxq_allocated -= extra; + sas->nb_repr_rxq += extra; + + /* Allocate extra representor TxQs up to the maximum */ + extra = MIN(evq_allocated, txq_allocated); + extra = MIN(extra, + SFC_REPR_PROXY_NB_TXQ_MAX - sas->nb_repr_txq); + evq_allocated -= extra; + txq_allocated -= extra; + sas->nb_repr_txq += extra; + } else { + sas->nb_repr_rxq = 0; + sas->nb_repr_txq = 0; + } + /* Add remaining allocated queues */ sa->rxq_max += MIN(rxq_allocated, evq_allocated / 2); sa->txq_max += MIN(txq_allocated, evq_allocated - sa->rxq_max); @@ -313,8 +354,10 @@ sfc_estimate_resource_limits(struct sfc_adapter *sa) static int sfc_set_drv_limits(struct sfc_adapter *sa) { + struct sfc_adapter_shared *sas = sfc_sa2shared(sa); const struct rte_eth_dev_data *data = sa->eth_dev->data; - uint32_t rxq_reserved = sfc_nb_reserved_rxq(sfc_sa2shared(sa)); + uint32_t rxq_reserved = sfc_nb_reserved_rxq(sas); + uint32_t txq_reserved = sfc_nb_txq_reserved(sas); efx_drv_limits_t lim; memset(&lim, 0, sizeof(lim)); @@ -325,10 +368,12 @@ sfc_set_drv_limits(struct sfc_adapter *sa) * sfc_estimate_resource_limits(). */ lim.edl_min_evq_count = lim.edl_max_evq_count = - 1 + data->nb_rx_queues + data->nb_tx_queues + rxq_reserved; + 1 + data->nb_rx_queues + data->nb_tx_queues + + rxq_reserved + txq_reserved; lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues + rxq_reserved; - lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; + lim.edl_min_txq_count = lim.edl_max_txq_count = + data->nb_tx_queues + txq_reserved; return efx_nic_set_drv_limits(sa->nic, &lim); } diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h index 8f65857f65..79f9d7979e 100644 --- a/drivers/net/sfc/sfc.h +++ b/drivers/net/sfc/sfc.h @@ -191,6 +191,8 @@ struct sfc_adapter_shared { char *dp_tx_name; bool counters_rxq_allocated; + unsigned int nb_repr_rxq; + unsigned int nb_repr_txq; }; /* Adapter process private data */ @@ -392,6 +394,19 @@ sfc_nb_counter_rxq(const struct sfc_adapter_shared *sas) } bool sfc_repr_supported(const struct sfc_adapter *sa); +bool sfc_repr_available(const struct sfc_adapter_shared *sas); + +static inline unsigned int +sfc_repr_nb_rxq(const struct sfc_adapter_shared *sas) +{ + return sas->nb_repr_rxq; +} + +static inline unsigned int +sfc_repr_nb_txq(const struct sfc_adapter_shared *sas) +{ + return sas->nb_repr_txq; +} /** Get the number of milliseconds since boot from the default timer */ static inline uint64_t diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h index b2a0380205..590cfb1694 100644 --- a/drivers/net/sfc/sfc_ev.h +++ b/drivers/net/sfc/sfc_ev.h @@ -70,14 +70,21 @@ sfc_mgmt_evq_sw_index(__rte_unused const struct sfc_adapter_shared *sas) static inline unsigned int sfc_nb_reserved_rxq(const struct sfc_adapter_shared *sas) { - return sfc_nb_counter_rxq(sas); + return sfc_nb_counter_rxq(sas) + sfc_repr_nb_rxq(sas); +} + +/* Return the number of Tx queues reserved for driver's internal use */ +static inline unsigned int +sfc_nb_txq_reserved(const struct sfc_adapter_shared *sas) +{ + return sfc_repr_nb_txq(sas); } static inline unsigned int sfc_nb_reserved_evq(const struct sfc_adapter_shared *sas) { - /* An EvQ is required for each reserved RxQ */ - return 1 + sfc_nb_reserved_rxq(sas); + /* An EvQ is required for each reserved Rx/Tx queue */ + return 1 + sfc_nb_reserved_rxq(sas) + sfc_nb_txq_reserved(sas); } /* @@ -112,6 +119,7 @@ sfc_counters_rxq_sw_index(const struct sfc_adapter_shared *sas) * Own event queue is allocated for management, each Rx and each Tx queue. * Zero event queue is used for management events. * When counters are supported, one Rx event queue is reserved. + * When representors are supported, Rx and Tx event queues are reserved. * Rx event queues follow reserved event queues. * Tx event queues follow Rx event queues. */ @@ -150,27 +158,37 @@ sfc_evq_sw_index_by_rxq_sw_index(struct sfc_adapter *sa, } static inline sfc_ethdev_qid_t -sfc_ethdev_tx_qid_by_txq_sw_index(__rte_unused struct sfc_adapter_shared *sas, +sfc_ethdev_tx_qid_by_txq_sw_index(struct sfc_adapter_shared *sas, sfc_sw_index_t txq_sw_index) { - /* Only ethdev queues are present for now */ - return txq_sw_index; + if (txq_sw_index < sfc_nb_txq_reserved(sas)) + return SFC_ETHDEV_QID_INVALID; + + return txq_sw_index - sfc_nb_txq_reserved(sas); } static inline sfc_sw_index_t -sfc_txq_sw_index_by_ethdev_tx_qid(__rte_unused struct sfc_adapter_shared *sas, +sfc_txq_sw_index_by_ethdev_tx_qid(struct sfc_adapter_shared *sas, sfc_ethdev_qid_t ethdev_qid) { - /* Only ethdev queues are present for now */ - return ethdev_qid; + return sfc_nb_txq_reserved(sas) + ethdev_qid; } static inline sfc_sw_index_t sfc_evq_sw_index_by_txq_sw_index(struct sfc_adapter *sa, sfc_sw_index_t txq_sw_index) { - return sfc_nb_reserved_evq(sfc_sa2shared(sa)) + - sa->eth_dev->data->nb_rx_queues + txq_sw_index; + struct sfc_adapter_shared *sas = sfc_sa2shared(sa); + sfc_ethdev_qid_t ethdev_qid; + + ethdev_qid = sfc_ethdev_tx_qid_by_txq_sw_index(sas, txq_sw_index); + if (ethdev_qid == SFC_ETHDEV_QID_INVALID) { + return sfc_nb_reserved_evq(sas) - sfc_nb_txq_reserved(sas) + + txq_sw_index; + } + + return sfc_nb_reserved_evq(sas) + sa->eth_dev->data->nb_rx_queues + + ethdev_qid; } int sfc_ev_attach(struct sfc_adapter *sa); diff --git a/drivers/net/sfc/sfc_repr_proxy.c b/drivers/net/sfc/sfc_repr_proxy.c index eb29376988..6d3962304f 100644 --- a/drivers/net/sfc/sfc_repr_proxy.c +++ b/drivers/net/sfc/sfc_repr_proxy.c @@ -29,6 +29,7 @@ sfc_repr_proxy_routine(void *arg) int sfc_repr_proxy_attach(struct sfc_adapter *sa) { + struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); struct sfc_repr_proxy *rp = &sa->repr_proxy; struct rte_service_spec service; uint32_t cid; @@ -37,7 +38,7 @@ sfc_repr_proxy_attach(struct sfc_adapter *sa) sfc_log_init(sa, "entry"); - if (!sfc_repr_supported(sa)) { + if (!sfc_repr_available(sas)) { sfc_log_init(sa, "representors not supported - skip"); return 0; } @@ -102,11 +103,12 @@ sfc_repr_proxy_attach(struct sfc_adapter *sa) void sfc_repr_proxy_detach(struct sfc_adapter *sa) { + struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); struct sfc_repr_proxy *rp = &sa->repr_proxy; sfc_log_init(sa, "entry"); - if (!sfc_repr_supported(sa)) { + if (!sfc_repr_available(sas)) { sfc_log_init(sa, "representors not supported - skip"); return; } @@ -120,6 +122,7 @@ sfc_repr_proxy_detach(struct sfc_adapter *sa) int sfc_repr_proxy_start(struct sfc_adapter *sa) { + struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); struct sfc_repr_proxy *rp = &sa->repr_proxy; int rc; @@ -129,7 +132,7 @@ sfc_repr_proxy_start(struct sfc_adapter *sa) * The condition to start the proxy is insufficient. It will be * complemented with representor port start/stop support. */ - if (!sfc_repr_supported(sa)) { + if (!sfc_repr_available(sas)) { sfc_log_init(sa, "representors not supported - skip"); return 0; } @@ -180,12 +183,13 @@ sfc_repr_proxy_start(struct sfc_adapter *sa) void sfc_repr_proxy_stop(struct sfc_adapter *sa) { + struct sfc_adapter_shared * const sas = sfc_sa2shared(sa); struct sfc_repr_proxy *rp = &sa->repr_proxy; int rc; sfc_log_init(sa, "entry"); - if (!sfc_repr_supported(sa)) { + if (!sfc_repr_available(sas)) { sfc_log_init(sa, "representors not supported - skip"); return; } diff --git a/drivers/net/sfc/sfc_repr_proxy.h b/drivers/net/sfc/sfc_repr_proxy.h index 40ce352335..953b9922c8 100644 --- a/drivers/net/sfc/sfc_repr_proxy.h +++ b/drivers/net/sfc/sfc_repr_proxy.h @@ -16,6 +16,14 @@ extern "C" { #endif +/* Number of supported RxQs with different mbuf memory pools */ +#define SFC_REPR_PROXY_NB_RXQ_MIN (1) +#define SFC_REPR_PROXY_NB_RXQ_MAX (1) + +/* One TxQ is required and sufficient for port representors support */ +#define SFC_REPR_PROXY_NB_TXQ_MIN (1) +#define SFC_REPR_PROXY_NB_TXQ_MAX (1) + struct sfc_repr_proxy { uint32_t service_core_id; uint32_t service_id; diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c index 49b239f4d2..c1b2e964f8 100644 --- a/drivers/net/sfc/sfc_tx.c +++ b/drivers/net/sfc/sfc_tx.c @@ -376,6 +376,8 @@ sfc_tx_configure(struct sfc_adapter *sa) const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues; + const unsigned int nb_rsvd_tx_queues = sfc_nb_txq_reserved(sas); + const unsigned int nb_txq_total = nb_tx_queues + nb_rsvd_tx_queues; int rc = 0; sfc_log_init(sa, "nb_tx_queues=%u (old %u)", @@ -395,11 +397,11 @@ sfc_tx_configure(struct sfc_adapter *sa) if (rc != 0) goto fail_check_mode; - if (nb_tx_queues == sas->txq_count) + if (nb_txq_total == sas->txq_count) goto done; if (sas->txq_info == NULL) { - sas->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues, + sas->txq_info = rte_calloc_socket("sfc-txqs", nb_txq_total, sizeof(sas->txq_info[0]), 0, sa->socket_id); if (sas->txq_info == NULL) @@ -410,7 +412,7 @@ sfc_tx_configure(struct sfc_adapter *sa) * since it should not be shared. */ rc = ENOMEM; - sa->txq_ctrl = calloc(nb_tx_queues, sizeof(sa->txq_ctrl[0])); + sa->txq_ctrl = calloc(nb_txq_total, sizeof(sa->txq_ctrl[0])); if (sa->txq_ctrl == NULL) goto fail_txqs_ctrl_alloc; } else { @@ -422,23 +424,23 @@ sfc_tx_configure(struct sfc_adapter *sa) new_txq_info = rte_realloc(sas->txq_info, - nb_tx_queues * sizeof(sas->txq_info[0]), 0); - if (new_txq_info == NULL && nb_tx_queues > 0) + nb_txq_total * sizeof(sas->txq_info[0]), 0); + if (new_txq_info == NULL && nb_txq_total > 0) goto fail_txqs_realloc; new_txq_ctrl = realloc(sa->txq_ctrl, - nb_tx_queues * sizeof(sa->txq_ctrl[0])); - if (new_txq_ctrl == NULL && nb_tx_queues > 0) + nb_txq_total * sizeof(sa->txq_ctrl[0])); + if (new_txq_ctrl == NULL && nb_txq_total > 0) goto fail_txqs_ctrl_realloc; sas->txq_info = new_txq_info; sa->txq_ctrl = new_txq_ctrl; - if (nb_tx_queues > sas->ethdev_txq_count) { - memset(&sas->txq_info[sas->ethdev_txq_count], 0, - (nb_tx_queues - sas->ethdev_txq_count) * + if (nb_txq_total > sas->txq_count) { + memset(&sas->txq_info[sas->txq_count], 0, + (nb_txq_total - sas->txq_count) * sizeof(sas->txq_info[0])); - memset(&sa->txq_ctrl[sas->ethdev_txq_count], 0, - (nb_tx_queues - sas->ethdev_txq_count) * + memset(&sa->txq_ctrl[sas->txq_count], 0, + (nb_txq_total - sas->txq_count) * sizeof(sa->txq_ctrl[0])); } } @@ -455,7 +457,8 @@ sfc_tx_configure(struct sfc_adapter *sa) sas->ethdev_txq_count++; } - sas->txq_count = sas->ethdev_txq_count; + /* TODO: initialize reserved queues when supported. */ + sas->txq_count = sas->ethdev_txq_count + nb_rsvd_tx_queues; done: return 0; -- 2.30.2