* Re: [dpdk-stable] [PATCH] net/mlx: support IOVA VA mode
2019-05-02 1:44 [dpdk-stable] [PATCH] net/mlx: support IOVA VA mode Yongseok Koh
@ 2019-05-02 7:31 ` Yongseok Koh
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Yongseok Koh
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 " Yongseok Koh
2 siblings, 0 replies; 9+ messages in thread
From: Yongseok Koh @ 2019-05-02 7:31 UTC (permalink / raw)
To: Shahaf Shuler; +Cc: dev, dpdk stable, Davide Caratti
> On May 1, 2019, at 6:44 PM, Yongseok Koh <yskoh@mellanox.com> wrote:
>
> Set RTE_PCI_DRV_IOVA_AS_VA to driver's drv_flags as device's IOMMU takes
> virtual address.
>
> Cc: stable@dpdk.org
> Cc: Davide Caratti <dcaratti@redhat.com>
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
Self nack.
pci_one_device_has_iova_va() checks RTE_KDRV_VFIO.
Will send out v2 with adding RTE_KDRV_NIC_MLX.
> drivers/net/mlx4/mlx4.c | 4 ++--
> drivers/net/mlx5/mlx5.c | 4 ++--
> 2 files changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
> index fe559c0407..eccb77b410 100644
> --- a/drivers/net/mlx4/mlx4.c
> +++ b/drivers/net/mlx4/mlx4.c
> @@ -1133,8 +1133,8 @@ static struct rte_pci_driver mlx4_driver = {
> },
> .id_table = mlx4_pci_id_map,
> .probe = mlx4_pci_probe,
> - .drv_flags = RTE_PCI_DRV_INTR_LSC |
> - RTE_PCI_DRV_INTR_RMV,
> + .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> + RTE_PCI_DRV_IOVA_AS_VA,
> };
>
> #ifdef RTE_IBVERBS_LINK_DLOPEN
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index de85e85300..ecab7f899f 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -2101,8 +2101,8 @@ static struct rte_pci_driver mlx5_driver = {
> .remove = mlx5_pci_remove,
> .dma_map = mlx5_dma_map,
> .dma_unmap = mlx5_dma_unmap,
> - .drv_flags = (RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> - RTE_PCI_DRV_PROBE_AGAIN),
> + .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> + RTE_PCI_DRV_PROBE_AGAIN | RTE_PCI_DRV_IOVA_AS_VA,
> };
>
> #ifdef RTE_IBVERBS_LINK_DLOPEN
> --
> 2.11.0
>
^ permalink raw reply [flat|nested] 9+ messages in thread* [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type
2019-05-02 1:44 [dpdk-stable] [PATCH] net/mlx: support IOVA VA mode Yongseok Koh
2019-05-02 7:31 ` Yongseok Koh
@ 2019-05-02 7:43 ` Yongseok Koh
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 2/2] net/mlx: support IOVA VA mode Yongseok Koh
2019-05-02 8:10 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Thomas Monjalon
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 " Yongseok Koh
2 siblings, 2 replies; 9+ messages in thread
From: Yongseok Koh @ 2019-05-02 7:43 UTC (permalink / raw)
To: ferruh.yigit, anatoly.burakov, thomas, shahafs; +Cc: dev, stable
Mellanox mlx4/5 PMD doesn't need to be detached from kernel driver and
attached to VFIO/UIO. Control path still goes through the existing kernel
drivers, which is mlx4_core/mlx5_core.
Cc: stable@dpdk.org
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
v2:
* add RTE_KDRV_NIC_MLX
drivers/bus/pci/linux/pci.c | 6 +++++-
lib/librte_eal/common/include/rte_dev.h | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index c99d523f0a..b931cf9d10 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -329,6 +329,9 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
dev->kdrv = RTE_KDRV_IGB_UIO;
else if (!strcmp(driver, "uio_pci_generic"))
dev->kdrv = RTE_KDRV_UIO_GENERIC;
+ else if (!strcmp(driver, "mlx4_core") ||
+ !strcmp(driver, "mlx5_core"))
+ dev->kdrv = RTE_KDRV_NIC_MLX;
else
dev->kdrv = RTE_KDRV_UNKNOWN;
} else
@@ -568,7 +571,8 @@ pci_one_device_has_iova_va(void)
FOREACH_DRIVER_ON_PCIBUS(drv) {
if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) {
FOREACH_DEVICE_ON_PCIBUS(dev) {
- if (dev->kdrv == RTE_KDRV_VFIO &&
+ if ((dev->kdrv == RTE_KDRV_VFIO ||
+ dev->kdrv == RTE_KDRV_NIC_MLX) &&
rte_pci_match(drv, dev))
return 1;
}
diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h
index 56dd52ad9e..a17023d88b 100644
--- a/lib/librte_eal/common/include/rte_dev.h
+++ b/lib/librte_eal/common/include/rte_dev.h
@@ -63,6 +63,7 @@ enum rte_kernel_driver {
RTE_KDRV_VFIO,
RTE_KDRV_UIO_GENERIC,
RTE_KDRV_NIC_UIO,
+ RTE_KDRV_NIC_MLX,
RTE_KDRV_NONE,
};
--
2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread* [dpdk-stable] [PATCH v2 2/2] net/mlx: support IOVA VA mode
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Yongseok Koh
@ 2019-05-02 7:43 ` Yongseok Koh
2019-05-02 8:10 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Thomas Monjalon
1 sibling, 0 replies; 9+ messages in thread
From: Yongseok Koh @ 2019-05-02 7:43 UTC (permalink / raw)
To: ferruh.yigit, anatoly.burakov, thomas, shahafs; +Cc: dev, stable
Set RTE_PCI_DRV_IOVA_AS_VA to driver's drv_flags as device's IOMMU takes
virtual address.
Cc: stable@dpdk.org
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
v2:
* add RTE_KDRV_NIC_MLX
drivers/net/mlx4/mlx4.c | 4 ++--
drivers/net/mlx5/mlx5.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index fe559c0407..eccb77b410 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -1133,8 +1133,8 @@ static struct rte_pci_driver mlx4_driver = {
},
.id_table = mlx4_pci_id_map,
.probe = mlx4_pci_probe,
- .drv_flags = RTE_PCI_DRV_INTR_LSC |
- RTE_PCI_DRV_INTR_RMV,
+ .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
+ RTE_PCI_DRV_IOVA_AS_VA,
};
#ifdef RTE_IBVERBS_LINK_DLOPEN
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index de85e85300..ecab7f899f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -2101,8 +2101,8 @@ static struct rte_pci_driver mlx5_driver = {
.remove = mlx5_pci_remove,
.dma_map = mlx5_dma_map,
.dma_unmap = mlx5_dma_unmap,
- .drv_flags = (RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
- RTE_PCI_DRV_PROBE_AGAIN),
+ .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
+ RTE_PCI_DRV_PROBE_AGAIN | RTE_PCI_DRV_IOVA_AS_VA,
};
#ifdef RTE_IBVERBS_LINK_DLOPEN
--
2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Yongseok Koh
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 2/2] net/mlx: support IOVA VA mode Yongseok Koh
@ 2019-05-02 8:10 ` Thomas Monjalon
1 sibling, 0 replies; 9+ messages in thread
From: Thomas Monjalon @ 2019-05-02 8:10 UTC (permalink / raw)
To: Yongseok Koh; +Cc: ferruh.yigit, anatoly.burakov, shahafs, dev, stable
02/05/2019 09:43, Yongseok Koh:
> Mellanox mlx4/5 PMD doesn't need to be detached from kernel driver and
> attached to VFIO/UIO. Control path still goes through the existing kernel
> drivers, which is mlx4_core/mlx5_core.
The real change here is to handle RTE_PCI_DRV_IOVA_AS_VA
with Mellanox kernel drivers.
It think it should be explained in the commit log,
because there is no other reason for not using RTE_KDRV_UNKNOWN.
> @@ -568,7 +571,8 @@ pci_one_device_has_iova_va(void)
> FOREACH_DRIVER_ON_PCIBUS(drv) {
> if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) {
> FOREACH_DEVICE_ON_PCIBUS(dev) {
> - if (dev->kdrv == RTE_KDRV_VFIO &&
> + if ((dev->kdrv == RTE_KDRV_VFIO ||
> + dev->kdrv == RTE_KDRV_NIC_MLX) &&
> rte_pci_match(drv, dev))
> return 1;
> }
^ permalink raw reply [flat|nested] 9+ messages in thread
* [dpdk-stable] [PATCH v3 1/2] bus/pci: add Mellanox kernel driver type
2019-05-02 1:44 [dpdk-stable] [PATCH] net/mlx: support IOVA VA mode Yongseok Koh
2019-05-02 7:31 ` Yongseok Koh
2019-05-02 7:43 ` [dpdk-stable] [PATCH v2 1/2] bus/pci: add Mellanox kernel driver type Yongseok Koh
@ 2019-05-02 9:07 ` Yongseok Koh
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 2/2] net/mlx: support IOVA VA mode Yongseok Koh
2019-06-03 22:39 ` [dpdk-stable] [dpdk-dev] [PATCH v3 1/2] bus/pci: add Mellanox kernel driver type Thomas Monjalon
2 siblings, 2 replies; 9+ messages in thread
From: Yongseok Koh @ 2019-05-02 9:07 UTC (permalink / raw)
To: ferruh.yigit, anatoly.burakov, thomas, shahafs; +Cc: dev, stable
When checking RTE_PCI_DRV_IOVA_AS_VA flag to determine IOVA mode,
pci_one_device_has_iova_va() returns true only if kernel driver of the
device is vfio. However, Mellanox mlx4/5 PMD doesn't need to be detached
from kernel driver and attached to VFIO/UIO. Control path still goes
through the existing kernel driver, which is mlx4_core/mlx5_core. In order
to make RTE_PCI_DRV_IOVA_AS_VA effective for mlx4/mlx5 PMD, a new kernel
driver type has to be introduced.
Cc: stable@dpdk.org
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
v3:
* make commit log more explanatory
v2:
* add RTE_KDRV_NIC_MLX
drivers/bus/pci/linux/pci.c | 6 +++++-
lib/librte_eal/common/include/rte_dev.h | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index c99d523f0a..b931cf9d10 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -329,6 +329,9 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
dev->kdrv = RTE_KDRV_IGB_UIO;
else if (!strcmp(driver, "uio_pci_generic"))
dev->kdrv = RTE_KDRV_UIO_GENERIC;
+ else if (!strcmp(driver, "mlx4_core") ||
+ !strcmp(driver, "mlx5_core"))
+ dev->kdrv = RTE_KDRV_NIC_MLX;
else
dev->kdrv = RTE_KDRV_UNKNOWN;
} else
@@ -568,7 +571,8 @@ pci_one_device_has_iova_va(void)
FOREACH_DRIVER_ON_PCIBUS(drv) {
if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) {
FOREACH_DEVICE_ON_PCIBUS(dev) {
- if (dev->kdrv == RTE_KDRV_VFIO &&
+ if ((dev->kdrv == RTE_KDRV_VFIO ||
+ dev->kdrv == RTE_KDRV_NIC_MLX) &&
rte_pci_match(drv, dev))
return 1;
}
diff --git a/lib/librte_eal/common/include/rte_dev.h b/lib/librte_eal/common/include/rte_dev.h
index 56dd52ad9e..a17023d88b 100644
--- a/lib/librte_eal/common/include/rte_dev.h
+++ b/lib/librte_eal/common/include/rte_dev.h
@@ -63,6 +63,7 @@ enum rte_kernel_driver {
RTE_KDRV_VFIO,
RTE_KDRV_UIO_GENERIC,
RTE_KDRV_NIC_UIO,
+ RTE_KDRV_NIC_MLX,
RTE_KDRV_NONE,
};
--
2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread* [dpdk-stable] [PATCH v3 2/2] net/mlx: support IOVA VA mode
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 " Yongseok Koh
@ 2019-05-02 9:07 ` Yongseok Koh
2019-05-02 13:06 ` [dpdk-stable] [dpdk-dev] " Shahaf Shuler
2019-06-03 22:39 ` [dpdk-stable] [dpdk-dev] [PATCH v3 1/2] bus/pci: add Mellanox kernel driver type Thomas Monjalon
1 sibling, 1 reply; 9+ messages in thread
From: Yongseok Koh @ 2019-05-02 9:07 UTC (permalink / raw)
To: ferruh.yigit, anatoly.burakov, thomas, shahafs; +Cc: dev, stable
Set RTE_PCI_DRV_IOVA_AS_VA to driver's drv_flags as device's IOMMU takes
virtual address.
Cc: stable@dpdk.org
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
v3:
* no change
v2:
* add RTE_KDRV_NIC_MLX
drivers/net/mlx4/mlx4.c | 4 ++--
drivers/net/mlx5/mlx5.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index fe559c0407..eccb77b410 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -1133,8 +1133,8 @@ static struct rte_pci_driver mlx4_driver = {
},
.id_table = mlx4_pci_id_map,
.probe = mlx4_pci_probe,
- .drv_flags = RTE_PCI_DRV_INTR_LSC |
- RTE_PCI_DRV_INTR_RMV,
+ .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
+ RTE_PCI_DRV_IOVA_AS_VA,
};
#ifdef RTE_IBVERBS_LINK_DLOPEN
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index de85e85300..ecab7f899f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -2101,8 +2101,8 @@ static struct rte_pci_driver mlx5_driver = {
.remove = mlx5_pci_remove,
.dma_map = mlx5_dma_map,
.dma_unmap = mlx5_dma_unmap,
- .drv_flags = (RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
- RTE_PCI_DRV_PROBE_AGAIN),
+ .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
+ RTE_PCI_DRV_PROBE_AGAIN | RTE_PCI_DRV_IOVA_AS_VA,
};
#ifdef RTE_IBVERBS_LINK_DLOPEN
--
2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [dpdk-stable] [dpdk-dev] [PATCH v3 2/2] net/mlx: support IOVA VA mode
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 2/2] net/mlx: support IOVA VA mode Yongseok Koh
@ 2019-05-02 13:06 ` Shahaf Shuler
0 siblings, 0 replies; 9+ messages in thread
From: Shahaf Shuler @ 2019-05-02 13:06 UTC (permalink / raw)
To: Yongseok Koh, ferruh.yigit, anatoly.burakov, Thomas Monjalon; +Cc: dev, stable
Thursday, May 2, 2019 12:08 PM, Yongseok Koh:
> Subject: [dpdk-dev] [PATCH v3 2/2] net/mlx: support IOVA VA mode
>
> Set RTE_PCI_DRV_IOVA_AS_VA to driver's drv_flags as device's IOMMU
> takes virtual address.
>
> Cc: stable@dpdk.org
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
> ---
>
> v3:
> * no change
>
> v2:
> * add RTE_KDRV_NIC_MLX
>
> drivers/net/mlx4/mlx4.c | 4 ++--
> drivers/net/mlx5/mlx5.c | 4 ++--
> 2 files changed, 4 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index
> fe559c0407..eccb77b410 100644
> --- a/drivers/net/mlx4/mlx4.c
> +++ b/drivers/net/mlx4/mlx4.c
> @@ -1133,8 +1133,8 @@ static struct rte_pci_driver mlx4_driver = {
> },
> .id_table = mlx4_pci_id_map,
> .probe = mlx4_pci_probe,
> - .drv_flags = RTE_PCI_DRV_INTR_LSC |
> - RTE_PCI_DRV_INTR_RMV,
> + .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> + RTE_PCI_DRV_IOVA_AS_VA,
> };
>
> #ifdef RTE_IBVERBS_LINK_DLOPEN
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> de85e85300..ecab7f899f 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -2101,8 +2101,8 @@ static struct rte_pci_driver mlx5_driver = {
> .remove = mlx5_pci_remove,
> .dma_map = mlx5_dma_map,
> .dma_unmap = mlx5_dma_unmap,
> - .drv_flags = (RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> - RTE_PCI_DRV_PROBE_AGAIN),
> + .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV |
> + RTE_PCI_DRV_PROBE_AGAIN |
> RTE_PCI_DRV_IOVA_AS_VA,
> };
>
> #ifdef RTE_IBVERBS_LINK_DLOPEN
> --
> 2.11.0
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [dpdk-stable] [dpdk-dev] [PATCH v3 1/2] bus/pci: add Mellanox kernel driver type
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 " Yongseok Koh
2019-05-02 9:07 ` [dpdk-stable] [PATCH v3 2/2] net/mlx: support IOVA VA mode Yongseok Koh
@ 2019-06-03 22:39 ` Thomas Monjalon
1 sibling, 0 replies; 9+ messages in thread
From: Thomas Monjalon @ 2019-06-03 22:39 UTC (permalink / raw)
To: Yongseok Koh; +Cc: dev, ferruh.yigit, anatoly.burakov, shahafs, stable
02/05/2019 11:07, Yongseok Koh:
> When checking RTE_PCI_DRV_IOVA_AS_VA flag to determine IOVA mode,
> pci_one_device_has_iova_va() returns true only if kernel driver of the
> device is vfio. However, Mellanox mlx4/5 PMD doesn't need to be detached
> from kernel driver and attached to VFIO/UIO. Control path still goes
> through the existing kernel driver, which is mlx4_core/mlx5_core. In order
> to make RTE_PCI_DRV_IOVA_AS_VA effective for mlx4/mlx5 PMD, a new kernel
> driver type has to be introduced.
>
> Cc: stable@dpdk.org
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Applied, thanks
^ permalink raw reply [flat|nested] 9+ messages in thread