From: "Sachin Saxena (OSS)" <sachin.saxena@oss.nxp.com>
To: rohit.raj@nxp.com, Hemant Agrawal <hemant.agrawal@nxp.com>
Cc: dev@dpdk.org
Subject: Re: [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API
Date: Wed, 5 Aug 2020 15:27:13 +0530 [thread overview]
Message-ID: <7be31514-7933-33a0-f50e-dcc6b71a2edb@oss.nxp.com> (raw)
In-Reply-To: <20200728164252.2000-3-rohit.raj@nxp.com>
Acked-by: Sachin Saxena<sachin.saxena@oss.nxp.com>
On 28-Jul-20 10:12 PM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
>
> This patch add support for closing the bus objects which
> were acquired In the bus probe.
>
> Some devices need to be cleaned while in both primary and
> secondary process and while some need to be cleaned only in
> case of primary process.
>
> The devices are closed as per the white list used while
> creating the objects in a particular process.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
> drivers/bus/fslmc/fslmc_bus.c | 15 +++-
> drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++-
> drivers/bus/fslmc/fslmc_vfio.h | 3 +-
> drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++-
> drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++-
> drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++
> drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 +
> drivers/bus/fslmc/rte_fslmc.h | 5 +-
> drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++-
> drivers/net/dpaa2/dpaa2_mux.c | 18 ++++-
> 10 files changed, 252 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
> index beb3dd008..9d3da6be3 100644
> --- a/drivers/bus/fslmc/fslmc_bus.c
> +++ b/drivers/bus/fslmc/fslmc_bus.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2016,2018-2019 NXP
> + * Copyright 2016,2018-2020 NXP
> *
> */
>
> @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
> return 1;
> }
>
> +static int
> +rte_fslmc_close(void)
> +{
> + int ret = 0;
> +
> + ret = fslmc_vfio_close_group();
> + if (ret)
> + DPAA2_BUS_ERR("Unable to close devices %d", ret);
> +
> + return 0;
> +}
> +
> static int
> rte_fslmc_probe(void)
> {
> @@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
> .bus = {
> .scan = rte_fslmc_scan,
> .probe = rte_fslmc_probe,
> + .close = rte_fslmc_close,
> .parse = rte_fslmc_parse,
> .find_device = rte_fslmc_find_device,
> .get_iommu_class = rte_dpaa2_get_iommu_class,
> diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
> index 9134ec552..34ccc9db3 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.c
> +++ b/drivers/bus/fslmc/fslmc_vfio.c
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - * Copyright 2016-2019 NXP
> + * Copyright 2016-2020 NXP
> *
> */
>
> @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
> return -1;
> }
>
> +static void
> +fslmc_close_iodevices(struct rte_dpaa2_device *dev)
> +{
> + struct rte_dpaa2_object *object = NULL;
> + struct rte_dpaa2_driver *drv;
> + int ret, probe_all;
> +
> + switch (dev->dev_type) {
> + case DPAA2_IO:
> + case DPAA2_CON:
> + case DPAA2_CI:
> + case DPAA2_BPOOL:
> + case DPAA2_MUX:
> + TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
> + if (dev->dev_type == object->dev_type)
> + object->close(dev->object_id);
> + else
> + continue;
> + }
> + break;
> + case DPAA2_ETH:
> + case DPAA2_CRYPTO:
> + case DPAA2_QDMA:
> + probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
> + RTE_BUS_SCAN_WHITELIST;
> + TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
> + if (drv->drv_type != dev->dev_type)
> + continue;
> + if (rte_dev_is_probed(&dev->device))
> + continue;
> + if (probe_all ||
> + (dev->device.devargs &&
> + dev->device.devargs->policy ==
> + RTE_DEV_WHITELISTED)) {
> + ret = drv->remove(dev);
> + if (ret)
> + DPAA2_BUS_ERR("Unable to remove");
> + }
> + }
> + break;
> + default:
> + break;
> + }
> +
> + DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
> + dev->device.name);
> +}
> +
> /*
> * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
> * EVENT) devices.
> @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
> return ret;
> }
>
> +int
> +fslmc_vfio_close_group(void)
> +{
> + struct rte_dpaa2_device *dev, *dev_temp;
> +
> + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
> + if (dev->device.devargs &&
> + dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
> + DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
> + dev->device.name);
> + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
> + continue;
> + }
> + switch (dev->dev_type) {
> + case DPAA2_ETH:
> + case DPAA2_CRYPTO:
> + case DPAA2_QDMA:
> + case DPAA2_IO:
> + fslmc_close_iodevices(dev);
> + break;
> + case DPAA2_CON:
> + case DPAA2_CI:
> + case DPAA2_BPOOL:
> + case DPAA2_MUX:
> + if (rte_eal_process_type() == RTE_PROC_SECONDARY)
> + continue;
> +
> + fslmc_close_iodevices(dev);
> + break;
> + case DPAA2_DPRTC:
> + default:
> + DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
> + dev->device.name);
> + }
> + }
> +
> + return 0;
> +}
> +
> int
> fslmc_vfio_process_group(void)
> {
> diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
> index bc7c6f62d..0a29fe739 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.h
> +++ b/drivers/bus/fslmc/fslmc_vfio.h
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - * Copyright 2016,2019 NXP
> + * Copyright 2016,2019-2020 NXP
> *
> */
>
> @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>
> int fslmc_vfio_setup_group(void);
> int fslmc_vfio_process_group(void);
> +int fslmc_vfio_close_group(void);
> char *fslmc_get_container(void);
> int fslmc_get_container_group(int *gropuid);
> int rte_fslmc_vfio_dmamap(void);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> index d9619848d..9a4cdfd7a 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
> *
> */
>
> @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
> static struct dpbp_dev_list dpbp_dev_list
> = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
>
> +static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
> +{
> + struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> + /* Get DPBP dev handle from list using index */
> + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
> + if (dpbp_dev->dpbp_id == dpbp_id)
> + break;
> + }
> +
> + return dpbp_dev;
> +}
> +
> static int
> dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
> return 0;
> }
>
> +static void
> +dpaa2_close_dpbp_device(int object_id)
> +{
> + struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> + dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
> +
> + if (dpbp_dev) {
> + dpaa2_free_dpbp_dev(dpbp_dev);
> + dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
> + TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
> + rte_free(dpbp_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
> .dev_type = DPAA2_BPOOL,
> .create = dpaa2_create_dpbp_device,
> + .close = dpaa2_close_dpbp_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> index d393ce618..9902f38ce 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2017 NXP
> + * Copyright 2017,2020 NXP
> *
> */
>
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
> static struct dpci_dev_list dpci_dev_list
> = TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
>
> +static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
> +{
> + struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> + /* Get DPCI dev handle from list using index */
> + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
> + if (dpci_dev->dpci_id == dpci_id)
> + break;
> + }
> +
> + return dpci_dev;
> +}
> +
> static int
> rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
> }
> }
>
> +
> +static void
> +rte_dpaa2_close_dpci_device(int object_id)
> +{
> + struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> + dpci_dev = get_dpci_from_id((uint32_t)object_id);
> +
> + if (dpci_dev) {
> + rte_dpaa2_free_dpci_dev(dpci_dev);
> + dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
> + TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
> + rte_free(dpci_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
> .dev_type = DPAA2_CI,
> .create = rte_dpaa2_create_dpci_device,
> + .close = rte_dpaa2_close_dpci_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> index b0055b164..b7caaea31 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> @@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
> * Cluster 4 (ID = x07) : CPU14, CPU15;
> */
>
> +static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
> +{
> + struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> + /* Get DPIO dev handle from list using index */
> + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
> + if (dpio_dev->hw_id == dpio_id)
> + break;
> + }
> +
> + return dpio_dev;
> +}
> +
> static int
> dpaa2_get_core_id(void)
> {
> @@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
> pthread_setspecific(dpaa2_portal_key, NULL);
> }
>
> +static void
> +dpaa2_close_dpio_device(int object_id)
> +{
> + struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> + dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
> +
> + if (dpio_dev) {
> + if (dpio_dev->dpio) {
> + dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
> + dpio_dev->token);
> + dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
> + dpio_dev->token);
> + rte_free(dpio_dev->dpio);
> + }
> + TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
> + rte_free(dpio_dev);
> + }
> +}
> +
> static int
> dpaa2_create_dpio_device(int vdev_fd,
> struct vfio_device_info *obj_info,
> @@ -628,6 +661,7 @@ dpaa2_free_eq_descriptors(void)
> static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
> .dev_type = DPAA2_IO,
> .create = dpaa2_create_dpio_device,
> + .close = dpaa2_close_dpio_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
> diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> index b169f5228..ac4ee2bf8 100644
> --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> @@ -30,6 +30,7 @@ INTERNAL {
> dpci_get_opr;
> dpci_set_opr;
> dpci_set_rx_queue;
> + dpcon_close;
> dpcon_get_attributes;
> dpcon_open;
> dpdmai_close;
> diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
> index 80873fffc..a938ab8f9 100644
> --- a/drivers/bus/fslmc/rte_fslmc.h
> +++ b/drivers/bus/fslmc/rte_fslmc.h
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2016,2019 NXP
> + * Copyright 2016,2019-2020 NXP
> *
> */
>
> @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
> struct vfio_device_info *obj_info,
> int object_id);
>
> +typedef void (*rte_dpaa2_obj_close_t)(int object_id);
> +
> /**
> * A structure describing a DPAA2 object.
> */
> @@ -87,6 +89,7 @@ struct rte_dpaa2_object {
> const char *name; /**< Name of Object. */
> enum rte_dpaa2_dev_type dev_type; /**< Type of device */
> rte_dpaa2_obj_create_t create;
> + rte_dpaa2_obj_close_t close;
> };
>
> /**
> diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> index 200b71640..86844f49f 100644
> --- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2017 NXP
> + * Copyright 2017,2020 NXP
> *
> */
>
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
> static struct dpcon_dev_list dpcon_dev_list
> = TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
>
> +static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
> +{
> + struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> + /* Get DPCONC dev handle from list using index */
> + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
> + if (dpcon_dev->dpcon_id == dpcon_id)
> + break;
> + }
> +
> + return dpcon_dev;
> +}
> +
> static int
> rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
> }
> }
>
> +
> +static void
> +rte_dpaa2_close_dpcon_device(int object_id)
> +{
> + struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> + dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
> +
> + if (dpcon_dev) {
> + rte_dpaa2_free_dpcon_dev(dpcon_dev);
> + dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
> + TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
> + rte_free(dpcon_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
> .dev_type = DPAA2_CON,
> .create = rte_dpaa2_create_dpcon_device,
> + .close = rte_dpaa2_close_dpcon_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
> diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
> index f8366e839..15114dac9 100644
> --- a/drivers/net/dpaa2/dpaa2_mux.c
> +++ b/drivers/net/dpaa2/dpaa2_mux.c
> @@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
> {
> struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
>
> - /* Get DPBP dev handle from list using index */
> + /* Get DPDMUX dev handle from list using index */
> TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
> if (dpdmux_dev->dpdmux_id == dpdmux_id)
> break;
> @@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
> return -1;
> }
>
> +static void
> +dpaa2_close_dpdmux_device(int object_id)
> +{
> + struct dpaa2_dpdmux_dev *dpdmux_dev;
> +
> + dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
> +
> + if (dpdmux_dev) {
> + dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
> + dpdmux_dev->token);
> + TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
> + rte_free(dpdmux_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
> .dev_type = DPAA2_MUX,
> .create = dpaa2_create_dpdmux_device,
> + .close = dpaa2_close_dpdmux_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
next prev parent reply other threads:[~2020-08-05 9:57 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-07-28 16:42 [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close rohit.raj
2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
2020-08-05 9:28 ` Sachin Saxena (OSS)
2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API rohit.raj
2020-08-05 9:57 ` Sachin Saxena (OSS) [this message]
2020-07-28 17:23 ` [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close Stephen Hemminger
2020-07-30 15:53 ` [dpdk-dev] [EXT] " Rohit Raj
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=7be31514-7933-33a0-f50e-dcc6b71a2edb@oss.nxp.com \
--to=sachin.saxena@oss.nxp.com \
--cc=dev@dpdk.org \
--cc=hemant.agrawal@nxp.com \
--cc=rohit.raj@nxp.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).