From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 832C8A00E6 for ; Thu, 21 Mar 2019 09:13:16 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A65871B489; Thu, 21 Mar 2019 09:12:11 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id A3E2A1B444 for ; Thu, 21 Mar 2019 09:11:42 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from viacheslavo@mellanox.com) with ESMTPS (AES256-SHA encrypted); 21 Mar 2019 10:11:37 +0200 Received: from pegasus12.mtr.labs.mlnx. (pegasus12.mtr.labs.mlnx [10.210.17.40]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2L8Bai8003643; Thu, 21 Mar 2019 10:11:37 +0200 From: Viacheslav Ovsiienko To: dev@dpdk.org Cc: shahafs@mellanox.com Date: Thu, 21 Mar 2019 08:11:24 +0000 Message-Id: <1553155888-27498-11-git-send-email-viacheslavo@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1553155888-27498-1-git-send-email-viacheslavo@mellanox.com> References: <1551376985-11096-1-git-send-email-viacheslavo@mellanox.com> <1553155888-27498-1-git-send-email-viacheslavo@mellanox.com> Subject: [dpdk-dev] [PATCH 10/14] net/mlx5: switch to the shared IB device context X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Content-Type: text/plain; charset="UTF-8" Message-ID: <20190321081124.YxYHlll2kLGOoIP6RgFVwR2bDyq6WlkMcig52fqYNX8@z> The code is updated to use the shared IB device context and device handles. The IB device context is shared between reprentors created over the single multiport IB device. All Verbs and DevX objects will be created whithin this shared context. Signed-off-by: Viacheslav Ovsiienko --- drivers/net/mlx5/mlx5.c | 7 +++---- drivers/net/mlx5/mlx5.h | 1 - drivers/net/mlx5/mlx5_ethdev.c | 13 +++++++------ drivers/net/mlx5/mlx5_flow_dv.c | 9 +++++---- drivers/net/mlx5/mlx5_flow_verbs.c | 6 ++++-- drivers/net/mlx5/mlx5_rxq.c | 23 ++++++++++++----------- drivers/net/mlx5/mlx5_socket.c | 4 ++-- drivers/net/mlx5/mlx5_trigger.c | 2 +- drivers/net/mlx5/mlx5_txq.c | 4 ++-- 9 files changed, 36 insertions(+), 33 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index a0f224b..312c42b 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -407,7 +407,7 @@ struct mlx5_dev_spawn_data { DRV_LOG(DEBUG, "port %u closing device \"%s\"", dev->data->port_id, - ((priv->ctx != NULL) ? priv->ctx->device->name : "")); + ((priv->sh->ctx != NULL) ? priv->sh->ctx->device->name : "")); /* In case mlx5_dev_stop() has not been called. */ mlx5_dev_interrupt_handler_uninstall(dev); mlx5_traffic_disable(dev); @@ -1111,7 +1111,6 @@ struct mlx5_dev_spawn_data { goto error; } priv->sh = sh; - priv->ctx = sh->ctx; priv->ibv_port = spawn->ibv_port; priv->mtu = ETHER_MTU; #ifndef RTE_ARCH_64 @@ -1349,7 +1348,7 @@ struct mlx5_dev_spawn_data { } } #ifdef HAVE_MLX5DV_DR - ns = mlx5dv_dr_create_ns(ctx, + ns = mlx5dv_dr_create_ns(sh->ctx, MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS); if (ns == NULL) { DRV_LOG(ERR, "mlx5dv_dr_create_ns failed"); @@ -1357,7 +1356,7 @@ struct mlx5_dev_spawn_data { goto error; } priv->rx_ns = ns; - ns = mlx5dv_dr_create_ns(ctx, + ns = mlx5dv_dr_create_ns(sh->ctx, MLX5DV_DR_NS_DOMAIN_EGRESS_BYPASS); if (ns == NULL) { DRV_LOG(ERR, "mlx5dv_dr_create_ns failed"); diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index efa3a3c..d816d24 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -226,7 +226,6 @@ struct mlx5_priv { struct rte_eth_dev_data *dev_data; /* Pointer to device data. */ struct mlx5_ibv_shared *sh; /* Shared IB device context. */ uint32_t ibv_port; /* IB device port number. */ - struct ibv_context *ctx; /* Verbs context. */ struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; /* MAC addresses. */ BITFIELD_DECLARE(mac_own, uint64_t, MLX5_MAX_MAC_ADDRESSES); /* Bit-field of MAC addresses owned by the PMD. */ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 69f238b..1b2173b 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -1055,7 +1055,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size) } /* Read all message and acknowledge them. */ for (;;) { - if (mlx5_glue->get_async_event(priv->ctx, &event)) + if (mlx5_glue->get_async_event(priv->sh->ctx, &event)) break; if ((event.event_type == IBV_EVENT_PORT_ACTIVE || event.event_type == IBV_EVENT_PORT_ERR) && @@ -1142,12 +1142,13 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size) mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; + struct ibv_context *ctx = priv->sh->ctx; int ret; int flags; - assert(priv->ctx->async_fd > 0); - flags = fcntl(priv->ctx->async_fd, F_GETFL); - ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + assert(ctx->async_fd > 0); + flags = fcntl(ctx->async_fd, F_GETFL); + ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK); if (ret) { DRV_LOG(INFO, "port %u failed to change file descriptor async event" @@ -1158,7 +1159,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size) } if (dev->data->dev_conf.intr_conf.lsc || dev->data->dev_conf.intr_conf.rmv) { - priv->intr_handle.fd = priv->ctx->async_fd; + priv->intr_handle.fd = ctx->async_fd; priv->intr_handle.type = RTE_INTR_HANDLE_EXT; rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); @@ -1303,7 +1304,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size) struct ibv_device_attr device_attr; struct mlx5_priv *priv = dev->data->dev_private; - if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO) + if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO) return 1; return 0; } diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 44b36ee..b8943da 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -840,7 +840,7 @@ struct field_modify_info modify_tcp[] = { *cache_resource = *resource; cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_packet_reformat - (priv->ctx, cache_resource->reformat_type, + (priv->sh->ctx, cache_resource->reformat_type, cache_resource->ft_type, ns, cache_resource->flags, cache_resource->size, (cache_resource->size ? cache_resource->buf : NULL)); @@ -1569,7 +1569,8 @@ struct field_modify_info modify_tcp[] = { *cache_resource = *resource; cache_resource->verbs_action = mlx5_glue->dv_create_flow_action_modify_header - (priv->ctx, cache_resource->ft_type, + (priv->sh->ctx, + cache_resource->ft_type, ns, 0, cache_resource->actions_num * sizeof(cache_resource->actions[0]), @@ -1629,7 +1630,7 @@ struct field_modify_info modify_tcp[] = { ret = -ENOMEM; goto error_exit; } - ret = mlx5_devx_cmd_flow_counter_alloc(priv->ctx, dcs); + ret = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, dcs); if (ret) goto error_exit; struct mlx5_flow_counter tmpl = { @@ -2982,7 +2983,7 @@ struct field_modify_info modify_tcp[] = { if (matcher->egress) dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS; cache_matcher->matcher_object = - mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr, + mlx5_glue->dv_create_flow_matcher(priv->sh->ctx, &dv_attr, tbl->obj); if (!cache_matcher->matcher_object) { rte_free(cache_matcher); diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 6c4f52f..9548d12 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -56,10 +56,11 @@ { #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) struct mlx5_priv *priv = dev->data->dev_private; + struct ibv_context ctx = priv->sh->ctx; struct ibv_counter_set_init_attr init = { .counter_set_id = counter->id}; - counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init); + counter->cs = mlx5_glue->create_counter_set(ctx, &init); if (!counter->cs) { rte_errno = ENOTSUP; return -ENOTSUP; @@ -67,12 +68,13 @@ return 0; #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) struct mlx5_priv *priv = dev->data->dev_private; + struct ibv_context *ctx = priv->sh->ctx; struct ibv_counters_init_attr init = {0}; struct ibv_counter_attach_attr attach; int ret; memset(&attach, 0, sizeof(attach)); - counter->cs = mlx5_glue->create_counters(priv->ctx, &init); + counter->cs = mlx5_glue->create_counters(ctx, &init); if (!counter->cs) { rte_errno = ENOTSUP; return -ENOTSUP; diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index fd1c3a2..dcb97c2 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -799,7 +799,7 @@ struct mlx5_rxq_ibv * } tmpl->rxq_ctrl = rxq_ctrl; if (rxq_ctrl->irq) { - tmpl->channel = mlx5_glue->create_comp_channel(priv->ctx); + tmpl->channel = mlx5_glue->create_comp_channel(priv->sh->ctx); if (!tmpl->channel) { DRV_LOG(ERR, "port %u: comp channel creation failure", dev->data->port_id); @@ -848,7 +848,7 @@ struct mlx5_rxq_ibv * } #endif tmpl->cq = mlx5_glue->cq_ex_to_cq - (mlx5_glue->dv_create_cq(priv->ctx, &attr.cq.ibv, + (mlx5_glue->dv_create_cq(priv->sh->ctx, &attr.cq.ibv, &attr.cq.mlx5)); if (tmpl->cq == NULL) { DRV_LOG(ERR, "port %u Rx queue %u CQ creation failure", @@ -905,10 +905,10 @@ struct mlx5_rxq_ibv * .two_byte_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT, }; } - tmpl->wq = mlx5_glue->dv_create_wq(priv->ctx, &attr.wq.ibv, + tmpl->wq = mlx5_glue->dv_create_wq(priv->sh->ctx, &attr.wq.ibv, &attr.wq.mlx5); #else - tmpl->wq = mlx5_glue->create_wq(priv->ctx, &attr.wq.ibv); + tmpl->wq = mlx5_glue->create_wq(priv->sh->ctx, &attr.wq.ibv); #endif if (tmpl->wq == NULL) { DRV_LOG(ERR, "port %u Rx queue %u WQ creation failure", @@ -1643,7 +1643,7 @@ struct mlx5_ind_table_ibv * for (j = 0; i != (unsigned int)(1 << wq_n); ++i, ++j) wq[i] = wq[j]; ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table - (priv->ctx, + (priv->sh->ctx, &(struct ibv_rwq_ind_table_init_attr){ .log_ind_tbl_size = wq_n, .ind_tbl = wq, @@ -1817,7 +1817,7 @@ struct mlx5_hrxq * } #endif qp = mlx5_glue->dv_create_qp - (priv->ctx, + (priv->sh->ctx, &(struct ibv_qp_init_attr_ex){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = @@ -1836,7 +1836,7 @@ struct mlx5_hrxq * &qp_init_attr); #else qp = mlx5_glue->create_qp_ex - (priv->ctx, + (priv->sh->ctx, &(struct ibv_qp_init_attr_ex){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = @@ -1988,20 +1988,21 @@ struct mlx5_rxq_ibv * mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; + struct ibv_context *ctx = priv->sh->ctx; struct ibv_cq *cq; struct ibv_wq *wq = NULL; struct mlx5_rxq_ibv *rxq; if (priv->drop_queue.rxq) return priv->drop_queue.rxq; - cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); + cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0); if (!cq) { DEBUG("port %u cannot allocate CQ for drop queue", dev->data->port_id); rte_errno = errno; goto error; } - wq = mlx5_glue->create_wq(priv->ctx, + wq = mlx5_glue->create_wq(ctx, &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, .max_wr = 1, @@ -2078,7 +2079,7 @@ struct mlx5_ind_table_ibv * if (!rxq) return NULL; tmpl.ind_table = mlx5_glue->create_rwq_ind_table - (priv->ctx, + (priv->sh->ctx, &(struct ibv_rwq_ind_table_init_attr){ .log_ind_tbl_size = 0, .ind_tbl = &rxq->wq, @@ -2145,7 +2146,7 @@ struct mlx5_hrxq * ind_tbl = mlx5_ind_table_ibv_drop_new(dev); if (!ind_tbl) return NULL; - qp = mlx5_glue->create_qp_ex(priv->ctx, + qp = mlx5_glue->create_qp_ex(priv->sh->ctx, &(struct ibv_qp_init_attr_ex){ .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c index 41cac3c..8fa6430 100644 --- a/drivers/net/mlx5/mlx5_socket.c +++ b/drivers/net/mlx5/mlx5_socket.c @@ -185,9 +185,9 @@ assert(cmsg != NULL); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd)); + cmsg->cmsg_len = CMSG_LEN(sizeof(priv->sh->ctx->cmd_fd)); fd = (int *)CMSG_DATA(cmsg); - *fd = priv->ctx->cmd_fd; + *fd = priv->sh->ctx->cmd_fd; ret = sendmsg(conn_sock, &msg, 0); if (ret < 0) DRV_LOG(WARNING, "port %u cannot send response", diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index 2137bdc..d13a1a1 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -58,7 +58,7 @@ goto error; } } - ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd); + ret = mlx5_tx_uar_remap(dev, priv->sh->ctx->cmd_fd); if (ret) { /* Adjust index for rollback. */ i = priv->txqs_n - 1; diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 10a3040..25305b6 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -392,7 +392,7 @@ struct mlx5_txq_ibv * ((desc / MLX5_TX_COMP_THRESH) - 1) : 1; if (is_empw_burst_func(tx_pkt_burst)) cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; - tmpl.cq = mlx5_glue->create_cq(priv->ctx, cqe_n, NULL, NULL, 0); + tmpl.cq = mlx5_glue->create_cq(priv->sh->ctx, cqe_n, NULL, NULL, 0); if (tmpl.cq == NULL) { DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure", dev->data->port_id, idx); @@ -435,7 +435,7 @@ struct mlx5_txq_ibv * attr.init.max_tso_header = txq_ctrl->max_tso_header; attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER; } - tmpl.qp = mlx5_glue->create_qp_ex(priv->ctx, &attr.init); + tmpl.qp = mlx5_glue->create_qp_ex(priv->sh->ctx, &attr.init); if (tmpl.qp == NULL) { DRV_LOG(ERR, "port %u Tx queue %u QP creation failure", dev->data->port_id, idx); -- 1.8.3.1