Re: [dpdk-dev] [PATCH v3 01/12] mempool/octeontx: probe timvf PCIe devices

[Jerin Jacob <jerin.jacob@caviumnetworks.com>]

-----Original Message-----
> Date: Tue,  3 Apr 2018 20:35:03 +0530
> From: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
> To: jerin.jacob@caviumnetworks.com, santosh.shukla@caviumnetworks.com,
>  erik.g.carrillo@intel.com
> Cc: dev@dpdk.org, Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
> Subject: [dpdk-dev] [PATCH v3 01/12] mempool/octeontx: probe timvf PCIe
>  devices
> X-Mailer: git-send-email 2.16.3
> 
> On Octeontx HW, each event timer device is enumerated as separate SRIOV VF
> PCIe device.
> 
> In order to expose as a event timer device:
> On PCIe probe, the driver stores the information associated with the
> PCIe device and later when appliacation requests for a event timer device
> through `rte_event_timer_adapter_create` the driver infrastructure creates
> the timer adapter with earlier probed PCIe VF devices.
> 
> Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
> ---
>  drivers/mempool/octeontx/Makefile                  |   1 +
>  drivers/mempool/octeontx/meson.build               |   1 +
>  drivers/mempool/octeontx/octeontx_mbox.h           |   7 +
>  drivers/mempool/octeontx/octeontx_timvf.c          | 145 +++++++++++++++++++++

Now that, we have driver/common for mailbox. Please move the
octeontx_timvf.c to driver/event/octeontx/timvf_probe.c

>  .../octeontx/rte_mempool_octeontx_version.map      |   3 +
>  5 files changed, 157 insertions(+)
>  create mode 100644 drivers/mempool/octeontx/octeontx_timvf.c
> 
> diff --git a/drivers/mempool/octeontx/Makefile b/drivers/mempool/octeontx/Makefile
> index dfc373e62..a0547437a 100644
> --- a/drivers/mempool/octeontx/Makefile
> +++ b/drivers/mempool/octeontx/Makefile
> @@ -20,6 +20,7 @@ LIBABIVER := 1
>  SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_ssovf.c
>  SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_mbox.c
>  SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_fpavf.c
> +SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += octeontx_timvf.c
>  SRCS-$(CONFIG_RTE_LIBRTE_OCTEONTX_MEMPOOL) += rte_mempool_octeontx.c
>  
>  ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
> diff --git a/drivers/mempool/octeontx/meson.build b/drivers/mempool/octeontx/meson.build
> index 1e894a56b..c938436c9 100644
> --- a/drivers/mempool/octeontx/meson.build
> +++ b/drivers/mempool/octeontx/meson.build
> @@ -4,6 +4,7 @@
>  sources = files('octeontx_ssovf.c',
>  		'octeontx_mbox.c',
>  		'octeontx_fpavf.c',
> +		'octeontx_timvf.c',
>  		'rte_mempool_octeontx.c'
>  )
>  
> diff --git a/drivers/mempool/octeontx/octeontx_mbox.h b/drivers/mempool/octeontx/octeontx_mbox.h
> index 1b056071a..7fcc9950f 100644
> --- a/drivers/mempool/octeontx/octeontx_mbox.h
> +++ b/drivers/mempool/octeontx/octeontx_mbox.h
> @@ -21,6 +21,11 @@ enum octeontx_ssovf_type {
>  	OCTEONTX_SSO_HWS,  /* SSO hardware workslot vf */
>  };
>  
> +struct octeontx_timvf_info {
> +	uint16_t domain; /* Domain id */
> +	uint8_t total_timvfs; /* Total timvf available in domain */
> +};
> +
>  struct octeontx_mbox_hdr {
>  	uint16_t vfid;  /* VF index or pf resource index local to the domain */
>  	uint8_t coproc; /* Coprocessor id */
> @@ -32,5 +37,7 @@ int octeontx_ssovf_info(struct octeontx_ssovf_info *info);
>  void *octeontx_ssovf_bar(enum octeontx_ssovf_type, uint8_t id, uint8_t bar);
>  int octeontx_ssovf_mbox_send(struct octeontx_mbox_hdr *hdr,
>  		void *txdata, uint16_t txlen, void *rxdata, uint16_t rxlen);
> +int octeontx_timvf_info(struct octeontx_timvf_info *info);
> +void *octeontx_timvf_bar(uint8_t id, uint8_t bar);
>  
>  #endif /* __OCTEONTX_MBOX_H__ */
> diff --git a/drivers/mempool/octeontx/octeontx_timvf.c b/drivers/mempool/octeontx/octeontx_timvf.c
> new file mode 100644
> index 000000000..b20743a33
> --- /dev/null
> +++ b/drivers/mempool/octeontx/octeontx_timvf.c
> @@ -0,0 +1,145 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2017 Cavium, Inc
> + */
> +
> +#include <rte_eal.h>
> +#include <rte_io.h>
> +#include <rte_pci.h>
> +#include <rte_bus_pci.h>
> +
> +#include "octeontx_mbox.h"
> +#include "octeontx_pool_logs.h"
> +
> +#ifndef PCI_VENDOR_ID_CAVIUM
> +#define PCI_VENDOR_ID_CAVIUM			(0x177D)
> +#endif
> +
> +#define PCI_DEVICE_ID_OCTEONTX_TIM_VF		(0xA051)
> +#define TIM_MAX_RINGS				(64)
> +
> +struct timvf_res {
> +	uint16_t domain;
> +	uint16_t vfid;
> +	void *bar0;
> +	void *bar2;
> +	void *bar4;
> +};
> +
> +struct timdev {
> +	uint8_t total_timvfs;
> +	struct timvf_res rings[TIM_MAX_RINGS];
> +};
> +
> +static struct timdev tdev;
> +
> +int
> +octeontx_timvf_info(struct octeontx_timvf_info *tinfo)
> +{
> +	int i;
> +	struct octeontx_ssovf_info info;
> +
> +	if (tinfo == NULL)
> +		return -EINVAL;
> +
> +	if (!tdev.total_timvfs)
> +		return -ENODEV;
> +
> +	if (octeontx_ssovf_info(&info) < 0)
> +		return -EINVAL;
> +
> +	for (i = 0; i < tdev.total_timvfs; i++) {
> +		if (info.domain != tdev.rings[i].domain) {
> +			mbox_log_err("GRP error, vfid=%d/%d domain=%d/%d %p",
> +				i, tdev.rings[i].vfid,
> +				info.domain, tdev.rings[i].domain,
> +				tdev.rings[i].bar0);
> +			return -EINVAL;
> +		}
> +	}
> +
> +	tinfo->total_timvfs = tdev.total_timvfs;
> +	tinfo->domain = info.domain;
> +	return 0;
> +}
> +
> +void*
> +octeontx_timvf_bar(uint8_t id, uint8_t bar)
> +{
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return NULL;
> +
> +	if (id > tdev.total_timvfs)
> +		return NULL;
> +
> +	switch (bar) {
> +	case 0:
> +		return tdev.rings[id].bar0;
> +	case 4:
> +		return tdev.rings[id].bar4;
> +	default:
> +		return NULL;
> +	}
> +}
> +
> +static int
> +timvf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
> +{
> +	uint64_t val;
> +	uint16_t vfid;
> +	struct timvf_res *res;
> +
> +	RTE_SET_USED(pci_drv);
> +
> +	/* For secondary processes, the primary has done all the work */
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return 0;
> +
> +	if (pci_dev->mem_resource[0].addr == NULL ||
> +			pci_dev->mem_resource[4].addr == NULL) {
> +		mbox_log_err("Empty bars %p %p",
> +				pci_dev->mem_resource[0].addr,
> +				pci_dev->mem_resource[4].addr);
> +		return -ENODEV;
> +	}
> +
> +	val = rte_read64((uint8_t *)pci_dev->mem_resource[0].addr + 0x100);

Remove 0x100 hard code or add extra comment, something like, 0x100 /* TIM_VRINGX_BASE */

> +	vfid = (val >> 23) & 0xff;
> +	if (vfid >= TIM_MAX_RINGS) {
> +		mbox_log_err("Invalid vfid(%d/%d)", vfid, TIM_MAX_RINGS);
> +		return -EINVAL;
> +	}
> +
> +	res = &tdev.rings[tdev.total_timvfs];
> +	res->vfid = vfid;
> +	res->bar0 = pci_dev->mem_resource[0].addr;
> +	res->bar2 = pci_dev->mem_resource[2].addr;
> +	res->bar4 = pci_dev->mem_resource[4].addr;
> +	res->domain = (val >> 7) & 0xffff;
> +	tdev.total_timvfs++;
> +	rte_wmb();
> +
> +	mbox_log_dbg("Domain=%d VFid=%d bar0 %p total_timvfs=%d", res->domain,
> +			res->vfid, pci_dev->mem_resource[0].addr,
> +			tdev.total_timvfs);
> +	return 0;
> +}
> +
> +
> +static const struct rte_pci_id pci_timvf_map[] = {
> +	{
> +		RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
> +				PCI_DEVICE_ID_OCTEONTX_TIM_VF)
> +	},
> +	{
> +		.vendor_id = 0,
> +	},
> +};
> +
> +static struct rte_pci_driver pci_timvf = {
> +	.id_table = pci_timvf_map,
> +	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
> +	.probe = timvf_probe,
> +	.remove = NULL,
> +};
> +
> +RTE_PMD_REGISTER_PCI(octeontx_timvf, pci_timvf);
> diff --git a/drivers/mempool/octeontx/rte_mempool_octeontx_version.map b/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
> index fe8cdeca0..91e0a3a93 100644
> --- a/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
> +++ b/drivers/mempool/octeontx/rte_mempool_octeontx_version.map
> @@ -5,5 +5,8 @@ DPDK_17.11 {
>  	octeontx_ssovf_bar;
>  	octeontx_ssovf_mbox_send;
>  
> +	octeontx_timvf_info;
> +	octeontx_timvf_bar;

We may not need to expose this once it is moved to driver/event/octeontx

With above changes:
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>


> +
>  	local: *;
>  };
> -- 
> 2.16.3
>