From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from stargate3.asicdesigners.com (stargate.chelsio.com [67.207.112.58]) by dpdk.org (Postfix) with ESMTP id AF5E4C494 for ; Mon, 29 Jun 2015 19:59:29 +0200 (CEST) Received: from localhost (scalar.blr.asicdesigners.com [10.193.185.94]) by stargate3.asicdesigners.com (8.13.8/8.13.8) with ESMTP id t5THxN8f003124; Mon, 29 Jun 2015 10:59:24 -0700 From: Rahul Lakkireddy To: dev@dpdk.org Date: Tue, 30 Jun 2015 04:58:36 +0530 Message-Id: X-Mailer: git-send-email 2.4.1 In-Reply-To: References: In-Reply-To: References: Cc: Felix Marti , Kumar Sanghvi , Nirranjan Kirubaharan Subject: [dpdk-dev] [PATCH v4 3/9] cxgbe: add device configuration and RX support for cxgbe PMD. X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 29 Jun 2015 17:59:32 -0000 Adds RX support for the cxgbe poll mode driver. This patch: 1. Adds rx queue related eth_dev_ops. 2. Adds RSS support. 3. Adds dev_configure() and dev_infos_get() eth_dev_ops. 4. Adds rx_pkt_burst for receiving packets. Signed-off-by: Rahul Lakkireddy Signed-off-by: Kumar Sanghvi --- v4: - Fix 32-bit compilation. v3: - No changes. v2: - This patch is a subset of patch 2/5 submitted in v1. - Cleanup some RX related macros and code. drivers/net/cxgbe/cxgbe.h | 6 + drivers/net/cxgbe/cxgbe_ethdev.c | 183 ++++++++ drivers/net/cxgbe/cxgbe_main.c | 350 +++++++++++++++ drivers/net/cxgbe/sge.c | 915 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 1454 insertions(+) diff --git a/drivers/net/cxgbe/cxgbe.h b/drivers/net/cxgbe/cxgbe.h index 44d48dc..90d1db0 100644 --- a/drivers/net/cxgbe/cxgbe.h +++ b/drivers/net/cxgbe/cxgbe.h @@ -44,5 +44,11 @@ #define CXGBE_DEFAULT_RX_DESC_SIZE 1024 /* Default RX ring size */ int cxgbe_probe(struct adapter *adapter); +void init_rspq(struct adapter *adap, struct sge_rspq *q, unsigned int us, + unsigned int cnt, unsigned int size, unsigned int iqe_size); +int setup_sge_fwevtq(struct adapter *adapter); +void cfg_queues(struct rte_eth_dev *eth_dev); +int cfg_queue_count(struct rte_eth_dev *eth_dev); +int setup_rss(struct port_info *pi); #endif /* _CXGBE_H_ */ diff --git a/drivers/net/cxgbe/cxgbe_ethdev.c b/drivers/net/cxgbe/cxgbe_ethdev.c index 30d39b4..1c69973 100644 --- a/drivers/net/cxgbe/cxgbe_ethdev.c +++ b/drivers/net/cxgbe/cxgbe_ethdev.c @@ -85,7 +85,189 @@ */ #include "t4_pci_id_tbl.h" +static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue; + unsigned int work_done; + + CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n", + __func__, rxq->rspq.cntxt_id, nb_pkts); + + if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done)) + dev_err(adapter, "error in cxgbe poll\n"); + + CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done); + return work_done; +} + +static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev, + struct rte_eth_dev_info *device_info) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adapter = pi->adapter; + int max_queues = adapter->sge.max_ethqsets / adapter->params.nports; + + device_info->min_rx_bufsize = 68; /* XXX: Smallest pkt size */ + device_info->max_rx_pktlen = 1500; /* XXX: For now we support mtu */ + device_info->max_rx_queues = max_queues; + device_info->max_tx_queues = max_queues; + device_info->max_mac_addrs = 1; + /* XXX: For now we support one MAC/port */ + device_info->max_vfs = adapter->params.arch.vfcount; + device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */ + + device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | + DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM; + + device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT | + DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM | + DEV_TX_OFFLOAD_TCP_TSO; + + device_info->reta_size = pi->rss_size; +} + +static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev, + uint16_t tx_queue_id); +static void cxgbe_dev_rx_queue_release(void *q); + +static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adapter = pi->adapter; + int err; + + CXGBE_FUNC_TRACE(); + + if (!(adapter->flags & FW_QUEUE_BOUND)) { + err = setup_sge_fwevtq(adapter); + if (err) + return err; + adapter->flags |= FW_QUEUE_BOUND; + } + + err = cfg_queue_count(eth_dev); + if (err) + return err; + + return 0; +} + +static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adap = pi->adapter; + struct sge_rspq *q; + + dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", + __func__, pi->port_id, rx_queue_id); + + q = eth_dev->data->rx_queues[rx_queue_id]; + return t4_sge_eth_rxq_start(adap, q); +} + +static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, + uint16_t rx_queue_id) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adap = pi->adapter; + struct sge_rspq *q; + + dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", + __func__, pi->port_id, rx_queue_id); + + q = eth_dev->data->rx_queues[rx_queue_id]; + return t4_sge_eth_rxq_stop(adap, q); +} + +static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, + uint16_t queue_idx, uint16_t nb_desc, + unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adapter = pi->adapter; + struct sge *s = &adapter->sge; + struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx]; + int err = 0; + int msi_idx = 0; + unsigned int temp_nb_desc; + + RTE_SET_USED(rx_conf); + + dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n", + __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc, + socket_id, mp); + + /* Free up the existing queue */ + if (eth_dev->data->rx_queues[queue_idx]) { + cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]); + eth_dev->data->rx_queues[queue_idx] = NULL; + } + + eth_dev->data->rx_queues[queue_idx] = (void *)rxq; + + /* Sanity Checking + * + * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE + */ + temp_nb_desc = nb_desc; + if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) { + dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n", + __func__, CXGBE_MIN_RING_DESC_SIZE, + CXGBE_DEFAULT_RX_DESC_SIZE); + temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE; + } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) { + dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n", + __func__, CXGBE_MIN_RING_DESC_SIZE, + CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE); + return -(EINVAL); + } + + rxq->rspq.size = temp_nb_desc; + if ((&rxq->fl) != NULL) + rxq->fl.size = temp_nb_desc; + + err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx, + &rxq->fl, t4_ethrx_handler, + t4_get_mps_bg_map(adapter, pi->tx_chan), mp, + queue_idx, socket_id); + + dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n", + __func__, err, pi->port_id, rxq->rspq.cntxt_id); + return err; +} + +static void cxgbe_dev_rx_queue_release(void *q) +{ + struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q; + struct sge_rspq *rq = &rxq->rspq; + + if (rq) { + struct port_info *pi = (struct port_info *) + (rq->eth_dev->data->dev_private); + struct adapter *adap = pi->adapter; + + dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n", + __func__, pi->port_id, rxq->rspq.cntxt_id); + + t4_sge_eth_rxq_release(adap, rxq); + } +} + static struct eth_dev_ops cxgbe_eth_dev_ops = { + .dev_configure = cxgbe_dev_configure, + .dev_infos_get = cxgbe_dev_info_get, + .rx_queue_setup = cxgbe_dev_rx_queue_setup, + .rx_queue_start = cxgbe_dev_rx_queue_start, + .rx_queue_stop = cxgbe_dev_rx_queue_stop, + .rx_queue_release = cxgbe_dev_rx_queue_release, }; /* @@ -103,6 +285,7 @@ static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev) CXGBE_FUNC_TRACE(); eth_dev->dev_ops = &cxgbe_eth_dev_ops; + eth_dev->rx_pkt_burst = &cxgbe_recv_pkts; /* for secondary processes, we don't initialise any further as primary * has already done this work. diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c index 6f3a6db..abcef6b 100644 --- a/drivers/net/cxgbe/cxgbe_main.c +++ b/drivers/net/cxgbe/cxgbe_main.c @@ -67,6 +67,249 @@ #include "t4_msg.h" #include "cxgbe.h" +/* + * Response queue handler for the FW event queue. + */ +static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, + __rte_unused const struct pkt_gl *gl) +{ + u8 opcode = ((const struct rss_header *)rsp)->opcode; + + rsp++; /* skip RSS header */ + + /* + * FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. + */ + if (unlikely(opcode == CPL_FW4_MSG && + ((const struct cpl_fw4_msg *)rsp)->type == + FW_TYPE_RSSCPL)) { + rsp++; + opcode = ((const struct rss_header *)rsp)->opcode; + rsp++; + if (opcode != CPL_SGE_EGR_UPDATE) { + dev_err(q->adapter, "unexpected FW4/CPL %#x on FW event queue\n", + opcode); + goto out; + } + } + + if (likely(opcode == CPL_SGE_EGR_UPDATE)) { + /* do nothing */ + } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { + const struct cpl_fw6_msg *msg = (const void *)rsp; + + t4_handle_fw_rpl(q->adapter, msg->data); + } else { + dev_err(adapter, "unexpected CPL %#x on FW event queue\n", + opcode); + } +out: + return 0; +} + +int setup_sge_fwevtq(struct adapter *adapter) +{ + struct sge *s = &adapter->sge; + int err = 0; + int msi_idx = 0; + + err = t4_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->eth_dev, + msi_idx, NULL, fwevtq_handler, -1, NULL, 0, + rte_socket_id()); + return err; +} + +static int closest_timer(const struct sge *s, int time) +{ + unsigned int i, match = 0; + int delta, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { + delta = time - s->timer_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + match = i; + } + } + return match; +} + +static int closest_thres(const struct sge *s, int thres) +{ + unsigned int i, match = 0; + int delta, min_delta = INT_MAX; + + for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { + delta = thres - s->counter_val[i]; + if (delta < 0) + delta = -delta; + if (delta < min_delta) { + min_delta = delta; + match = i; + } + } + return match; +} + +/** + * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters + * @q: the Rx queue + * @us: the hold-off time in us, or 0 to disable timer + * @cnt: the hold-off packet count, or 0 to disable counter + * + * Sets an Rx queue's interrupt hold-off time and packet count. At least + * one of the two needs to be enabled for the queue to generate interrupts. + */ +int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us, + unsigned int cnt) +{ + struct adapter *adap = q->adapter; + unsigned int timer_val; + + if (cnt) { + int err; + u32 v, new_idx; + + new_idx = closest_thres(&adap->sge, cnt); + if (q->desc && q->pktcnt_idx != new_idx) { + /* the queue has already been created, update it */ + v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | + V_FW_PARAMS_PARAM_X( + FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | + V_FW_PARAMS_PARAM_YZ(q->cntxt_id); + err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, + &v, &new_idx); + if (err) + return err; + } + q->pktcnt_idx = new_idx; + } + + timer_val = (us == 0) ? X_TIMERREG_RESTART_COUNTER : + closest_timer(&adap->sge, us); + + if ((us | cnt) == 0) + q->intr_params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); + else + q->intr_params = V_QINTR_TIMER_IDX(timer_val) | + V_QINTR_CNT_EN(cnt > 0); + return 0; +} + +static inline bool is_x_1g_port(const struct link_config *lc) +{ + return ((lc->supported & FW_PORT_CAP_SPEED_1G) != 0); +} + +static inline bool is_x_10g_port(const struct link_config *lc) +{ + return ((lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || + (lc->supported & FW_PORT_CAP_SPEED_40G) != 0 || + (lc->supported & FW_PORT_CAP_SPEED_100G) != 0); +} + +inline void init_rspq(struct adapter *adap, struct sge_rspq *q, + unsigned int us, unsigned int cnt, + unsigned int size, unsigned int iqe_size) +{ + q->adapter = adap; + cxgb4_set_rspq_intr_params(q, us, cnt); + q->iqe_len = iqe_size; + q->size = size; +} + +int cfg_queue_count(struct rte_eth_dev *eth_dev) +{ + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adap = pi->adapter; + struct sge *s = &adap->sge; + unsigned int max_queues = s->max_ethqsets / adap->params.nports; + + if ((eth_dev->data->nb_rx_queues < 1) || + (eth_dev->data->nb_tx_queues < 1)) + return -EINVAL; + + if ((eth_dev->data->nb_rx_queues > max_queues) || + (eth_dev->data->nb_tx_queues > max_queues)) + return -EINVAL; + + if (eth_dev->data->nb_rx_queues > pi->rss_size) + return -EINVAL; + + /* We must configure RSS, since config has changed*/ + pi->flags &= ~PORT_RSS_DONE; + + pi->n_rx_qsets = eth_dev->data->nb_rx_queues; + pi->n_tx_qsets = eth_dev->data->nb_tx_queues; + + return 0; +} + +void cfg_queues(struct rte_eth_dev *eth_dev) +{ + struct rte_config *config = rte_eal_get_configuration(); + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + struct adapter *adap = pi->adapter; + struct sge *s = &adap->sge; + unsigned int i, nb_ports = 0, qidx = 0; + unsigned int q_per_port = 0; + + if (!(adap->flags & CFG_QUEUES)) { + for_each_port(adap, i) { + struct port_info *tpi = adap2pinfo(adap, i); + + nb_ports += (is_x_10g_port(&tpi->link_cfg)) || + is_x_1g_port(&tpi->link_cfg) ? 1 : 0; + } + + /* + * We default up to # of cores queues per 1G/10G port. + */ + if (nb_ports) + q_per_port = (MAX_ETH_QSETS - + (adap->params.nports - nb_ports)) / + nb_ports; + + if (q_per_port > config->lcore_count) + q_per_port = config->lcore_count; + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + pi->first_qset = qidx; + + /* Initially n_rx_qsets == n_tx_qsets */ + pi->n_rx_qsets = (is_x_10g_port(&pi->link_cfg) || + is_x_1g_port(&pi->link_cfg)) ? + q_per_port : 1; + pi->n_tx_qsets = pi->n_rx_qsets; + + if (pi->n_rx_qsets > pi->rss_size) + pi->n_rx_qsets = pi->rss_size; + + qidx += pi->n_rx_qsets; + } + + s->max_ethqsets = qidx; + + for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { + struct sge_eth_rxq *r = &s->ethrxq[i]; + + init_rspq(adap, &r->rspq, 0, 0, 1024, 64); + r->usembufs = 1; + r->fl.size = (r->usembufs ? 1024 : 72); + } + + for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) + s->ethtxq[i].q.size = 1024; + + init_rspq(adap, &adap->sge.fw_evtq, 0, 0, 1024, 64); + adap->flags |= CFG_QUEUES; + } +} + static void setup_memwin(struct adapter *adap) { u32 mem_win0_base; @@ -89,6 +332,25 @@ static void setup_memwin(struct adapter *adap) MEMWIN_NIC)); } +static int init_rss(struct adapter *adap) +{ + unsigned int i; + int err; + + err = t4_init_rss_mode(adap, adap->mbox); + if (err) + return err; + + for_each_port(adap, i) { + struct port_info *pi = adap2pinfo(adap, i); + + pi->rss = rte_zmalloc(NULL, pi->rss_size, 0); + if (!pi->rss) + return -ENOMEM; + } + return 0; +} + static void print_port_info(struct adapter *adap) { int i; @@ -564,6 +826,87 @@ void t4_os_portmod_changed(const struct adapter *adap, int port_id) pi->port_id, pi->mod_type); } +/** + * cxgb4_write_rss - write the RSS table for a given port + * @pi: the port + * @queues: array of queue indices for RSS + * + * Sets up the portion of the HW RSS table for the port's VI to distribute + * packets to the Rx queues in @queues. + */ +int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) +{ + u16 *rss; + int i, err; + struct adapter *adapter = pi->adapter; + const struct sge_eth_rxq *rxq; + + /* Should never be called before setting up sge eth rx queues */ + BUG_ON(!(adapter->flags & FULL_INIT_DONE)); + + rxq = &adapter->sge.ethrxq[pi->first_qset]; + rss = rte_zmalloc(NULL, pi->rss_size * sizeof(u16), 0); + if (!rss) + return -ENOMEM; + + /* map the queue indices to queue ids */ + for (i = 0; i < pi->rss_size; i++, queues++) + rss[i] = rxq[*queues].rspq.abs_id; + + err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0, + pi->rss_size, rss, pi->rss_size); + /* + * If Tunnel All Lookup isn't specified in the global RSS + * Configuration, then we need to specify a default Ingress + * Queue for any ingress packets which aren't hashed. We'll + * use our first ingress queue ... + */ + if (!err) + err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid, + F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | + F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | + F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | + F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | + F_FW_RSS_VI_CONFIG_CMD_UDPEN, + rss[0]); + rte_free(rss); + return err; +} + +/** + * setup_rss - configure RSS + * @adapter: the adapter + * + * Sets up RSS to distribute packets to multiple receive queues. We + * configure the RSS CPU lookup table to distribute to the number of HW + * receive queues, and the response queue lookup table to narrow that + * down to the response queues actually configured for each port. + * We always configure the RSS mapping for all ports since the mapping + * table has plenty of entries. + */ +int setup_rss(struct port_info *pi) +{ + int j, err; + struct adapter *adapter = pi->adapter; + + dev_debug(adapter, "%s: pi->rss_size = %u; pi->n_rx_qsets = %u\n", + __func__, pi->rss_size, pi->n_rx_qsets); + + if (!pi->flags & PORT_RSS_DONE) { + if (adapter->flags & FULL_INIT_DONE) { + /* Fill default values with equal distribution */ + for (j = 0; j < pi->rss_size; j++) + pi->rss[j] = j % pi->n_rx_qsets; + + err = cxgb4_write_rss(pi, pi->rss); + if (err) + return err; + pi->flags |= PORT_RSS_DONE; + } + } + return 0; +} + int cxgbe_probe(struct adapter *adapter) { struct port_info *pi; @@ -662,6 +1005,7 @@ allocate_mac: pi->eth_dev->data->dev_private = pi; pi->eth_dev->driver = adapter->eth_dev->driver; pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops; + pi->eth_dev->rx_pkt_burst = adapter->eth_dev->rx_pkt_burst; TAILQ_INIT(&pi->eth_dev->link_intr_cbs); pi->eth_dev->data->mac_addrs = rte_zmalloc(name, @@ -683,8 +1027,14 @@ allocate_mac: } } + cfg_queues(adapter->eth_dev); + print_port_info(adapter); + err = init_rss(adapter); + if (err) + goto out_free; + return 0; out_free: diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c index 20da0fd..c7abd8d 100644 --- a/drivers/net/cxgbe/sge.c +++ b/drivers/net/cxgbe/sge.c @@ -69,6 +69,13 @@ #include "cxgbe.h" /* + * Max number of Rx buffers we replenish at a time. + */ +#define MAX_RX_REFILL 16U + +#define NOMEM_TMR_IDX (SGE_NTIMERS - 1) + +/* * Rx buffer sizes for "usembufs" Free List buffers (one ingress packet * per mbuf buffer). We currently only support two sizes for 1500- and * 9000-byte MTUs. We could easily support more but there doesn't seem to be @@ -118,6 +125,914 @@ enum { }; /** + * fl_cap - return the capacity of a free-buffer list + * @fl: the FL + * + * Returns the capacity of a free-buffer list. The capacity is less than + * the size because one descriptor needs to be left unpopulated, otherwise + * HW will think the FL is empty. + */ +static inline unsigned int fl_cap(const struct sge_fl *fl) +{ + return fl->size - 8; /* 1 descriptor = 8 buffers */ +} + +/** + * fl_starving - return whether a Free List is starving. + * @adapter: pointer to the adapter + * @fl: the Free List + * + * Tests specified Free List to see whether the number of buffers + * available to the hardware has falled below our "starvation" + * threshold. + */ +static inline bool fl_starving(const struct adapter *adapter, + const struct sge_fl *fl) +{ + const struct sge *s = &adapter->sge; + + return fl->avail - fl->pend_cred <= s->fl_starve_thres; +} + +static inline unsigned int get_buf_size(struct adapter *adapter, + const struct rx_sw_desc *d) +{ + struct sge *s = &adapter->sge; + unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE; + unsigned int buf_size; + + switch (rx_buf_size_idx) { + case RX_SMALL_PG_BUF: + buf_size = PAGE_SIZE; + break; + + case RX_LARGE_PG_BUF: + buf_size = PAGE_SIZE << s->fl_pg_order; + break; + + case RX_SMALL_MTU_BUF: + buf_size = FL_MTU_SMALL_BUFSIZE(adapter); + break; + + case RX_LARGE_MTU_BUF: + buf_size = FL_MTU_LARGE_BUFSIZE(adapter); + break; + + default: + BUG_ON(1); + buf_size = 0; /* deal with bogus compiler warnings */ + /* NOTREACHED */ + } + + return buf_size; +} + +/** + * free_rx_bufs - free the Rx buffers on an SGE free list + * @q: the SGE free list to free buffers from + * @n: how many buffers to free + * + * Release the next @n buffers on an SGE free-buffer Rx queue. The + * buffers must be made inaccessible to HW before calling this function. + */ +static void free_rx_bufs(struct sge_fl *q, int n) +{ + unsigned int cidx = q->cidx; + struct rx_sw_desc *d; + + d = &q->sdesc[cidx]; + while (n--) { + if (d->buf) { + rte_pktmbuf_free(d->buf); + d->buf = NULL; + } + ++d; + if (++cidx == q->size) { + cidx = 0; + d = q->sdesc; + } + q->avail--; + } + q->cidx = cidx; +} + +/** + * unmap_rx_buf - unmap the current Rx buffer on an SGE free list + * @q: the SGE free list + * + * Unmap the current buffer on an SGE free-buffer Rx queue. The + * buffer must be made inaccessible to HW before calling this function. + * + * This is similar to @free_rx_bufs above but does not free the buffer. + * Do note that the FL still loses any further access to the buffer. + */ +static void unmap_rx_buf(struct sge_fl *q) +{ + if (++q->cidx == q->size) + q->cidx = 0; + q->avail--; +} + +static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) +{ + if (q->pend_cred >= 8) { + u32 val = adap->params.arch.sge_fl_db; + + if (is_t4(adap->params.chip)) + val |= V_PIDX(q->pend_cred / 8); + else + val |= V_PIDX_T5(q->pend_cred / 8); + + /* + * Make sure all memory writes to the Free List queue are + * committed before we tell the hardware about them. + */ + wmb(); + + /* + * If we don't have access to the new User Doorbell (T5+), use + * the old doorbell mechanism; otherwise use the new BAR2 + * mechanism. + */ + if (unlikely(!q->bar2_addr)) { + t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + val | V_QID(q->cntxt_id)); + } else { + writel(val | V_QID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + + SGE_UDB_KDOORBELL)); + + /* + * This Write memory Barrier will force the write to + * the User Doorbell area to be flushed. + */ + wmb(); + } + q->pend_cred &= 7; + } +} + +static inline struct rte_mbuf *cxgbe_rxmbuf_alloc(struct rte_mempool *mp) +{ + struct rte_mbuf *m; + + m = __rte_mbuf_raw_alloc(mp); + __rte_mbuf_sanity_check_raw(m, 0); + return m; +} + +static inline void set_rx_sw_desc(struct rx_sw_desc *sd, void *buf, + dma_addr_t mapping) +{ + sd->buf = buf; + sd->dma_addr = mapping; /* includes size low bits */ +} + +/** + * refill_fl_usembufs - refill an SGE Rx buffer ring with mbufs + * @adap: the adapter + * @q: the ring to refill + * @n: the number of new buffers to allocate + * + * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, + * allocated with the supplied gfp flags. The caller must assure that + * @n does not exceed the queue's capacity. If afterwards the queue is + * found critically low mark it as starving in the bitmap of starving FLs. + * + * Returns the number of buffers allocated. + */ +static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, + int n) +{ + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, fl); + unsigned int cred = q->avail; + __be64 *d = &q->desc[q->pidx]; + struct rx_sw_desc *sd = &q->sdesc[q->pidx]; + unsigned int buf_size_idx = RX_SMALL_MTU_BUF; + + while (n--) { + struct rte_mbuf *mbuf = cxgbe_rxmbuf_alloc(rxq->rspq.mb_pool); + dma_addr_t mapping; + + if (!mbuf) { + dev_debug(adap, "%s: mbuf alloc failed\n", __func__); + q->alloc_failed++; + rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++; + goto out; + } + + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->next = NULL; + + mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off); + + mapping |= buf_size_idx; + *d++ = cpu_to_be64(mapping); + set_rx_sw_desc(sd, mbuf, mapping); + sd++; + + q->avail++; + if (++q->pidx == q->size) { + q->pidx = 0; + sd = q->sdesc; + d = q->desc; + } + } + +out: cred = q->avail - cred; + q->pend_cred += cred; + ring_fl_db(adap, q); + + if (unlikely(fl_starving(adap, q))) { + /* + * Make sure data has been written to free list + */ + wmb(); + q->low++; + } + + return cred; +} + +/** + * refill_fl - refill an SGE Rx buffer ring with mbufs + * @adap: the adapter + * @q: the ring to refill + * @n: the number of new buffers to allocate + * + * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, + * allocated with the supplied gfp flags. The caller must assure that + * @n does not exceed the queue's capacity. Returns the number of buffers + * allocated. + */ +static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n) +{ + return refill_fl_usembufs(adap, q, n); +} + +static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) +{ + refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail)); +} + +/** + * alloc_ring - allocate resources for an SGE descriptor ring + * @dev: the PCI device's core device + * @nelem: the number of descriptors + * @elem_size: the size of each descriptor + * @sw_size: the size of the SW state associated with each ring element + * @phys: the physical address of the allocated ring + * @metadata: address of the array holding the SW state for the ring + * @stat_size: extra space in HW ring for status information + * @node: preferred node for memory allocations + * + * Allocates resources for an SGE descriptor ring, such as Tx queues, + * free buffer lists, or response queues. Each SGE ring requires + * space for its HW descriptors plus, optionally, space for the SW state + * associated with each HW entry (the metadata). The function returns + * three values: the virtual address for the HW ring (the return value + * of the function), the bus address of the HW ring, and the address + * of the SW ring. + */ +static void *alloc_ring(size_t nelem, size_t elem_size, + size_t sw_size, dma_addr_t *phys, void *metadata, + size_t stat_size, __rte_unused uint16_t queue_id, + int socket_id, const char *z_name, + const char *z_name_sw) +{ + size_t len = CXGBE_MAX_RING_DESC_SIZE * elem_size + stat_size; + const struct rte_memzone *tz; + void *s = NULL; + + dev_debug(adapter, "%s: nelem = %zu; elem_size = %zu; sw_size = %zu; " + "stat_size = %zu; queue_id = %u; socket_id = %d; z_name = %s;" + " z_name_sw = %s\n", __func__, nelem, elem_size, sw_size, + stat_size, queue_id, socket_id, z_name, z_name_sw); + + tz = rte_memzone_lookup(z_name); + if (tz) { + dev_debug(adapter, "%s: tz exists...returning existing..\n", + __func__); + goto alloc_sw_ring; + } + + /* + * Allocate TX/RX ring hardware descriptors. A memzone large enough to + * handle the maximum ring size is allocated in order to allow for + * resizing in later calls to the queue setup function. + */ + tz = rte_memzone_reserve_aligned(z_name, len, socket_id, 0, 4096); + if (!tz) + return NULL; + +alloc_sw_ring: + memset(tz->addr, 0, len); + if (sw_size) { + s = rte_zmalloc_socket(z_name_sw, nelem * sw_size, + RTE_CACHE_LINE_SIZE, socket_id); + + if (!s) { + dev_err(adapter, "%s: failed to get sw_ring memory\n", + __func__); + return NULL; + } + } + if (metadata) + *(void **)metadata = s; + + *phys = (uint64_t)tz->phys_addr; + return tz->addr; +} + +/** + * t4_pktgl_to_mbuf_usembufs - build an mbuf from a packet gather list + * @gl: the gather list + * + * Builds an mbuf from the given packet gather list. Returns the mbuf or + * %NULL if mbuf allocation failed. + */ +static struct rte_mbuf *t4_pktgl_to_mbuf_usembufs(const struct pkt_gl *gl) +{ + /* + * If there's only one mbuf fragment, just return that. + */ + if (likely(gl->nfrags == 1)) + return gl->mbufs[0]; + + return NULL; +} + +/** + * t4_pktgl_to_mbuf - build an mbuf from a packet gather list + * @gl: the gather list + * + * Builds an mbuf from the given packet gather list. Returns the mbuf or + * %NULL if mbuf allocation failed. + */ +static struct rte_mbuf *t4_pktgl_to_mbuf(const struct pkt_gl *gl) +{ + return t4_pktgl_to_mbuf_usembufs(gl); +} + +#define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \ + ((dma_addr_t) ((mb)->buf_physaddr + (mb)->data_off)) + +/** + * t4_ethrx_handler - process an ingress ethernet packet + * @q: the response queue that received the packet + * @rsp: the response queue descriptor holding the RX_PKT message + * @si: the gather list of packet fragments + * + * Process an ingress ethernet packet and deliver it to the stack. + */ +int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, + const struct pkt_gl *si) +{ + struct rte_mbuf *mbuf; + const struct cpl_rx_pkt *pkt; + const struct rss_header *rss_hdr; + bool csum_ok; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + + rss_hdr = (const void *)rsp; + pkt = (const void *)&rsp[1]; + csum_ok = pkt->csum_calc && !pkt->err_vec; + + mbuf = t4_pktgl_to_mbuf(si); + if (unlikely(!mbuf)) { + rxq->stats.rx_drops++; + return 0; + } + + mbuf->port = pkt->iff; + if (pkt->l2info & htonl(F_RXF_IP)) { + mbuf->ol_flags |= PKT_RX_IPV4_HDR; + if (unlikely(!csum_ok)) + mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if ((pkt->l2info & htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; + } else if (pkt->l2info & htonl(F_RXF_IP6)) { + mbuf->ol_flags |= PKT_RX_IPV6_HDR; + } + + mbuf->port = pkt->iff; + + if (!rss_hdr->filter_tid && rss_hdr->hash_type) { + mbuf->ol_flags |= PKT_RX_RSS_HASH; + mbuf->hash.rss = ntohl(rss_hdr->hash_val); + } + + if (pkt->vlan_ex) { + mbuf->ol_flags |= PKT_RX_VLAN_PKT; + mbuf->vlan_tci = ntohs(pkt->vlan); + } + rxq->stats.pkts++; + rxq->stats.rx_bytes += mbuf->pkt_len; + + return 0; +} + +/** + * restore_rx_bufs - put back a packet's Rx buffers + * @q: the SGE free list + * @frags: number of FL buffers to restore + * + * Puts back on an FL the Rx buffers. The buffers have already been + * unmapped and are left unmapped, we mark them so to prevent further + * unmapping attempts. + * + * This function undoes a series of @unmap_rx_buf calls when we find out + * that the current packet can't be processed right away afterall and we + * need to come back to it later. This is a very rare event and there's + * no effort to make this particularly efficient. + */ +static void restore_rx_bufs(struct sge_fl *q, int frags) +{ + while (frags--) { + if (q->cidx == 0) + q->cidx = q->size - 1; + else + q->cidx--; + q->avail++; + } +} + +/** + * is_new_response - check if a response is newly written + * @r: the response descriptor + * @q: the response queue + * + * Returns true if a response descriptor contains a yet unprocessed + * response. + */ +static inline bool is_new_response(const struct rsp_ctrl *r, + const struct sge_rspq *q) +{ + return (r->u.type_gen >> S_RSPD_GEN) == q->gen; +} + +#define CXGB4_MSG_AN ((void *)1) + +/** + * rspq_next - advance to the next entry in a response queue + * @q: the queue + * + * Updates the state of a response queue to advance it to the next entry. + */ +static inline void rspq_next(struct sge_rspq *q) +{ + q->cur_desc = (const __be64 *)((const char *)q->cur_desc + q->iqe_len); + if (unlikely(++q->cidx == q->size)) { + q->cidx = 0; + q->gen ^= 1; + q->cur_desc = q->desc; + } +} + +/** + * process_responses - process responses from an SGE response queue + * @q: the ingress queue to process + * @budget: how many responses can be processed in this round + * @rx_pkts: mbuf to put the pkts + * + * Process responses from an SGE response queue up to the supplied budget. + * Responses include received packets as well as control messages from FW + * or HW. + * + * Additionally choose the interrupt holdoff time for the next interrupt + * on this queue. If the system is under memory shortage use a fairly + * long delay to help recovery. + */ +static int process_responses(struct sge_rspq *q, int budget, + struct rte_mbuf **rx_pkts) +{ + int ret = 0, rsp_type; + int budget_left = budget; + const struct rsp_ctrl *rc; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + struct adapter *adapter = q->adapter; + + while (likely(budget_left)) { + rc = (const struct rsp_ctrl *) + ((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc))); + + if (!is_new_response(rc, q)) + break; + + /* + * Ensure response has been read + */ + rmb(); + rsp_type = G_RSPD_TYPE(rc->u.type_gen); + + if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) { + struct pkt_gl si; + const struct rx_sw_desc *rsd; + struct rte_mbuf *pkt = NULL; + u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags; + + si.usembufs = rxq->usembufs; + /* + * In "use mbufs" mode, we don't pack multiple + * ingress packets per buffer (mbuf) so we + * should _always_ get a "New Buffer" flags + * from the SGE. Also, since we hand the + * mbuf's up to the host stack for it to + * eventually free, we don't release the mbuf's + * in the driver (in contrast to the "packed + * page" mode where the driver needs to + * release its reference on the page buffers). + */ + BUG_ON(!(len & F_RSPD_NEWBUF)); + len = G_RSPD_LEN(len); + si.tot_len = len; + + /* gather packet fragments */ + for (frags = 0; len; frags++) { + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + bufsz = min(get_buf_size(adapter, rsd), len); + pkt = rsd->buf; + pkt->data_len = bufsz; + pkt->pkt_len = bufsz; + si.mbufs[frags] = pkt; + len -= bufsz; + unmap_rx_buf(&rxq->fl); + } + + si.va = RTE_PTR_ADD(si.mbufs[0]->buf_addr, + si.mbufs[0]->data_off); + rte_prefetch1(si.va); + + /* + * For the "use mbuf" case here, we can end up + * chewing through our Free List very rapidly + * with one entry per Ingress packet getting + * consumed. So if the handler() successfully + * consumed the mbuf, check to see if we can + * refill the Free List incrementally in the + * loop ... + */ + si.nfrags = frags; + ret = q->handler(q, q->cur_desc, &si); + + if (unlikely(ret != 0)) { + restore_rx_bufs(&rxq->fl, frags); + } else { + rx_pkts[budget - budget_left] = pkt; + if (fl_cap(&rxq->fl) - rxq->fl.avail >= 8) + __refill_fl(q->adapter, &rxq->fl); + } + + } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) { + ret = q->handler(q, q->cur_desc, NULL); + } else { + ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN); + } + + if (unlikely(ret)) { + /* couldn't process descriptor, back off for recovery */ + q->next_intr_params = V_QINTR_TIMER_IDX(NOMEM_TMR_IDX); + break; + } + + rspq_next(q); + budget_left--; + } + + /* + * If this is a Response Queue with an associated Free List and + * there's room for another chunk of new Free List buffer pointers, + * refill the Free List. + */ + + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 8) + __refill_fl(q->adapter, &rxq->fl); + + return budget - budget_left; +} + +int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts, + unsigned int budget, unsigned int *work_done) +{ + unsigned int params; + u32 val; + int err = 0; + + *work_done = process_responses(q, budget, rx_pkts); + params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); + q->next_intr_params = params; + val = V_CIDXINC(*work_done) | V_SEINTARM(params); + + if (*work_done) { + /* + * If we don't have access to the new User GTS (T5+), + * use the old doorbell mechanism; otherwise use the new + * BAR2 mechanism. + */ + if (unlikely(!q->bar2_addr)) + t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), + val | V_INGRESSQID((u32)q->cntxt_id)); + else { + writel(val | V_INGRESSQID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + + SGE_UDB_GTS)); + /* + * This Write memory Barrier will force the write to + * the User Doorbell area to be flushed. + */ + wmb(); + } + } + + return err; +} + +/** + * bar2_address - return the BAR2 address for an SGE Queue's Registers + * @adapter: the adapter + * @qid: the SGE Queue ID + * @qtype: the SGE Queue Type (Egress or Ingress) + * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues + * + * Returns the BAR2 address for the SGE Queue Registers associated with + * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also + * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE + * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID" + * Registers are supported (e.g. the Write Combining Doorbell Buffer). + */ +static void __iomem *bar2_address(struct adapter *adapter, unsigned int qid, + enum t4_bar2_qtype qtype, + unsigned int *pbar2_qid) +{ + u64 bar2_qoffset; + int ret; + + ret = t4_bar2_sge_qregs(adapter, qid, qtype, &bar2_qoffset, pbar2_qid); + if (ret) + return NULL; + + return adapter->bar2 + bar2_qoffset; +} + +int t4_sge_eth_rxq_start(struct adapter *adap, struct sge_rspq *rq) +{ + struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq); + unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff; + + return t4_iq_start_stop(adap, adap->mbox, true, adap->pf, 0, + rq->cntxt_id, fl_id, 0xffff); +} + +int t4_sge_eth_rxq_stop(struct adapter *adap, struct sge_rspq *rq) +{ + struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq); + unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff; + + return t4_iq_start_stop(adap, adap->mbox, false, adap->pf, 0, + rq->cntxt_id, fl_id, 0xffff); +} + +/* + * @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0 + * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map + */ +int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, + struct rte_eth_dev *eth_dev, int intr_idx, + struct sge_fl *fl, rspq_handler_t hnd, int cong, + struct rte_mempool *mp, int queue_id, int socket_id) +{ + int ret, flsz = 0; + struct fw_iq_cmd c; + struct sge *s = &adap->sge; + struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private); + char z_name[RTE_MEMZONE_NAMESIZE]; + char z_name_sw[RTE_MEMZONE_NAMESIZE]; + unsigned int nb_refill; + + /* Size needs to be multiple of 16, including status entry. */ + iq->size = roundup(iq->size, 16); + + snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", + eth_dev->driver->pci_drv.name, fwevtq ? "fwq_ring" : "rx_ring", + eth_dev->data->port_id, queue_id); + snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); + + iq->desc = alloc_ring(iq->size, iq->iqe_len, 0, &iq->phys_addr, NULL, 0, + queue_id, socket_id, z_name, z_name_sw); + if (!iq->desc) + return -ENOMEM; + + memset(&c, 0, sizeof(c)); + c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | + F_FW_CMD_WRITE | F_FW_CMD_EXEC | + V_FW_IQ_CMD_PFN(adap->pf) | V_FW_IQ_CMD_VFN(0)); + c.alloc_to_len16 = htonl(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | + (sizeof(c) / 16)); + c.type_to_iqandstindex = + htonl(V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | + V_FW_IQ_CMD_IQASYNCH(fwevtq) | + V_FW_IQ_CMD_VIID(pi->viid) | + V_FW_IQ_CMD_IQANDST(intr_idx < 0) | + V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT) | + V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx : + -intr_idx - 1)); + c.iqdroprss_to_iqesize = + htons(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + F_FW_IQ_CMD_IQGTSMODE | + V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) | + V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4)); + c.iqsize = htons(iq->size); + c.iqaddr = cpu_to_be64(iq->phys_addr); + if (cong >= 0) + c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN); + + if (fl) { + struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, + fl); + enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip); + + /* + * Allocate the ring for the hardware free list (with space + * for its status page) along with the associated software + * descriptor ring. The free list size needs to be a multiple + * of the Egress Queue Unit and at least 2 Egress Units larger + * than the SGE's Egress Congrestion Threshold + * (fl_starve_thres - 1). + */ + if (fl->size < s->fl_starve_thres - 1 + 2 * 8) + fl->size = s->fl_starve_thres - 1 + 2 * 8; + fl->size = roundup(fl->size, 8); + + snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", + eth_dev->driver->pci_drv.name, + fwevtq ? "fwq_ring" : "fl_ring", + eth_dev->data->port_id, queue_id); + snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); + + fl->desc = alloc_ring(fl->size, sizeof(__be64), + sizeof(struct rx_sw_desc), + &fl->addr, &fl->sdesc, s->stat_len, + queue_id, socket_id, z_name, z_name_sw); + + if (!fl->desc) + goto fl_nomem; + + flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc); + c.iqns_to_fl0congen |= + htonl(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | + (unlikely(rxq->usembufs) ? + 0 : F_FW_IQ_CMD_FL0PACKEN) | + F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | + F_FW_IQ_CMD_FL0PADEN); + if (cong >= 0) + c.iqns_to_fl0congen |= + htonl(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | + F_FW_IQ_CMD_FL0CONGCIF | + F_FW_IQ_CMD_FL0CONGEN); + + /* In T6, for egress queue type FL there is internal overhead + * of 16B for header going into FLM module. + * Hence maximum allowed burst size will be 448 bytes. + */ + c.fl0dcaen_to_fl0cidxfthresh = + htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) | + V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ? + X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); + c.fl0size = htons(flsz); + c.fl0addr = cpu_to_be64(fl->addr); + } + + ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c); + if (ret) + goto err; + + iq->cur_desc = iq->desc; + iq->cidx = 0; + iq->gen = 1; + iq->next_intr_params = iq->intr_params; + iq->cntxt_id = ntohs(c.iqid); + iq->abs_id = ntohs(c.physiqid); + iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS, + &iq->bar2_qid); + iq->size--; /* subtract status entry */ + iq->eth_dev = eth_dev; + iq->handler = hnd; + iq->mb_pool = mp; + + /* set offset to -1 to distinguish ingress queues without FL */ + iq->offset = fl ? 0 : -1; + + if (fl) { + fl->cntxt_id = ntohs(c.fl0id); + fl->avail = 0; + fl->pend_cred = 0; + fl->pidx = 0; + fl->cidx = 0; + fl->alloc_failed = 0; + + /* + * Note, we must initialize the BAR2 Free List User Doorbell + * information before refilling the Free List! + */ + fl->bar2_addr = bar2_address(adap, fl->cntxt_id, + T4_BAR2_QTYPE_EGRESS, + &fl->bar2_qid); + + nb_refill = refill_fl(adap, fl, fl_cap(fl)); + if (nb_refill != fl_cap(fl)) { + ret = -ENOMEM; + dev_err(adap, "%s: mbuf alloc failed with error: %d\n", + __func__, ret); + goto refill_fl_err; + } + } + + /* + * For T5 and later we attempt to set up the Congestion Manager values + * of the new RX Ethernet Queue. This should really be handled by + * firmware because it's more complex than any host driver wants to + * get involved with and it's different per chip and this is almost + * certainly wrong. Formware would be wrong as well, but it would be + * a lot easier to fix in one place ... For now we do something very + * simple (and hopefully less wrong). + */ + if (!is_t4(adap->params.chip) && cong >= 0) { + u32 param, val; + int i; + + param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | + V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | + V_FW_PARAMS_PARAM_YZ(iq->cntxt_id)); + if (cong == 0) { + val = V_CONMCTXT_CNGTPMODE(X_CONMCTXT_CNGTPMODE_QUEUE); + } else { + val = V_CONMCTXT_CNGTPMODE( + X_CONMCTXT_CNGTPMODE_CHANNEL); + for (i = 0; i < 4; i++) { + if (cong & (1 << i)) + val |= V_CONMCTXT_CNGCHMAP(1 << + (i << 2)); + } + } + ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, + ¶m, &val); + if (ret) + dev_warn(adap->pdev_dev, "Failed to set Congestion Manager Context for Ingress Queue %d: %d\n", + iq->cntxt_id, -ret); + } + + return 0; + +refill_fl_err: + t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, + iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); +fl_nomem: + ret = -ENOMEM; +err: + iq->cntxt_id = 0; + iq->abs_id = 0; + if (iq->desc) + iq->desc = NULL; + + if (fl && fl->desc) { + rte_free(fl->sdesc); + fl->cntxt_id = 0; + fl->sdesc = NULL; + fl->desc = NULL; + } + return ret; +} + +static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, + struct sge_fl *fl) +{ + unsigned int fl_id = fl ? fl->cntxt_id : 0xffff; + + t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, + rq->cntxt_id, fl_id, 0xffff); + rq->cntxt_id = 0; + rq->abs_id = 0; + rq->desc = NULL; + + if (fl) { + free_rx_bufs(fl, fl->avail); + rte_free(fl->sdesc); + fl->sdesc = NULL; + fl->cntxt_id = 0; + fl->desc = NULL; + } +} + +void t4_sge_eth_rxq_release(struct adapter *adap, struct sge_eth_rxq *rxq) +{ + if (rxq->rspq.desc) { + t4_sge_eth_rxq_stop(adap, &rxq->rspq); + free_rspq_fl(adap, &rxq->rspq, rxq->fl.size ? &rxq->fl : NULL); + } +} + +/** * t4_sge_init - initialize SGE * @adap: the adapter * -- 2.4.1