Re: [dpdk-dev] [PATCH v5 21/35] event/cnxk: add event port and queue xstats

["Kinsella, Ray" <mdr@ashroe.eu>]

On 04/05/2021 01:27, pbhagavatula@marvell.com wrote:
> From: Pavan Nikhilesh <pbhagavatula@marvell.com>
> 
> Add support for retrieving statistics from SSO HWS and HWGRP.
> 
> Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
> ---
>  drivers/common/cnxk/roc_sso.c            |  63 +++++
>  drivers/common/cnxk/roc_sso.h            |  19 ++
>  drivers/common/cnxk/version.map          |   2 +
>  drivers/event/cnxk/cnxk_eventdev.h       |  15 ++
>  drivers/event/cnxk/cnxk_eventdev_stats.c | 289 +++++++++++++++++++++++
>  drivers/event/cnxk/meson.build           |   3 +-
>  6 files changed, 390 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/event/cnxk/cnxk_eventdev_stats.c
> 
> diff --git a/drivers/common/cnxk/roc_sso.c b/drivers/common/cnxk/roc_sso.c
> index 80d032039..1ccf2626b 100644
> --- a/drivers/common/cnxk/roc_sso.c
> +++ b/drivers/common/cnxk/roc_sso.c
> @@ -279,6 +279,69 @@ roc_sso_hws_unlink(struct roc_sso *roc_sso, uint8_t hws, uint16_t hwgrp[],
>  	return nb_hwgrp;
>  }
>  
> +int
> +roc_sso_hws_stats_get(struct roc_sso *roc_sso, uint8_t hws,
> +		      struct roc_sso_hws_stats *stats)
> +{
> +	struct dev *dev = &roc_sso_to_sso_priv(roc_sso)->dev;
> +	struct sso_hws_stats *req_rsp;
> +	int rc;
> +
> +	req_rsp = (struct sso_hws_stats *)mbox_alloc_msg_sso_hws_get_stats(
> +		dev->mbox);
> +	if (req_rsp == NULL) {
> +		rc = mbox_process(dev->mbox);
> +		if (rc < 0)
> +			return rc;
> +		req_rsp = (struct sso_hws_stats *)
> +			mbox_alloc_msg_sso_hws_get_stats(dev->mbox);
> +		if (req_rsp == NULL)
> +			return -ENOSPC;
> +	}
> +	req_rsp->hws = hws;
> +	rc = mbox_process_msg(dev->mbox, (void **)&req_rsp);
> +	if (rc)
> +		return rc;
> +
> +	stats->arbitration = req_rsp->arbitration;
> +	return 0;
> +}
> +
> +int
> +roc_sso_hwgrp_stats_get(struct roc_sso *roc_sso, uint8_t hwgrp,
> +			struct roc_sso_hwgrp_stats *stats)
> +{
> +	struct dev *dev = &roc_sso_to_sso_priv(roc_sso)->dev;
> +	struct sso_grp_stats *req_rsp;
> +	int rc;
> +
> +	req_rsp = (struct sso_grp_stats *)mbox_alloc_msg_sso_grp_get_stats(
> +		dev->mbox);
> +	if (req_rsp == NULL) {
> +		rc = mbox_process(dev->mbox);
> +		if (rc < 0)
> +			return rc;
> +		req_rsp = (struct sso_grp_stats *)
> +			mbox_alloc_msg_sso_grp_get_stats(dev->mbox);
> +		if (req_rsp == NULL)
> +			return -ENOSPC;
> +	}
> +	req_rsp->grp = hwgrp;
> +	rc = mbox_process_msg(dev->mbox, (void **)&req_rsp);
> +	if (rc)
> +		return rc;
> +
> +	stats->aw_status = req_rsp->aw_status;
> +	stats->dq_pc = req_rsp->dq_pc;
> +	stats->ds_pc = req_rsp->ds_pc;
> +	stats->ext_pc = req_rsp->ext_pc;
> +	stats->page_cnt = req_rsp->page_cnt;
> +	stats->ts_pc = req_rsp->ts_pc;
> +	stats->wa_pc = req_rsp->wa_pc;
> +	stats->ws_pc = req_rsp->ws_pc;
> +	return 0;
> +}
> +
>  int
>  roc_sso_hwgrp_hws_link_status(struct roc_sso *roc_sso, uint8_t hws,
>  			      uint16_t hwgrp)
> diff --git a/drivers/common/cnxk/roc_sso.h b/drivers/common/cnxk/roc_sso.h
> index f85799ba8..a6030e7d8 100644
> --- a/drivers/common/cnxk/roc_sso.h
> +++ b/drivers/common/cnxk/roc_sso.h
> @@ -12,6 +12,21 @@ struct roc_sso_hwgrp_qos {
>  	uint8_t taq_prcnt;
>  };
>  
> +struct roc_sso_hws_stats {
> +	uint64_t arbitration;
> +};
> +
> +struct roc_sso_hwgrp_stats {
> +	uint64_t ws_pc;
> +	uint64_t ext_pc;
> +	uint64_t wa_pc;
> +	uint64_t ts_pc;
> +	uint64_t ds_pc;
> +	uint64_t dq_pc;
> +	uint64_t aw_status;
> +	uint64_t page_cnt;
> +};
> +
>  struct roc_sso {
>  	struct plt_pci_device *pci_dev;
>  	/* Public data. */
> @@ -61,5 +76,9 @@ uintptr_t __roc_api roc_sso_hwgrp_base_get(struct roc_sso *roc_sso,
>  /* Debug */
>  void __roc_api roc_sso_dump(struct roc_sso *roc_sso, uint8_t nb_hws,
>  			    uint16_t hwgrp, FILE *f);
> +int __roc_api roc_sso_hwgrp_stats_get(struct roc_sso *roc_sso, uint8_t hwgrp,
> +				      struct roc_sso_hwgrp_stats *stats);
Missing rte_internal?
> +int __roc_api roc_sso_hws_stats_get(struct roc_sso *roc_sso, uint8_t hws,
> +				    struct roc_sso_hws_stats *stats);
Missing rte_internal?
>  
>  #endif /* _ROC_SSOW_H_ */
> diff --git a/drivers/common/cnxk/version.map b/drivers/common/cnxk/version.map
> index 5f2264f23..8e67c83a6 100644
> --- a/drivers/common/cnxk/version.map
> +++ b/drivers/common/cnxk/version.map
> @@ -183,8 +183,10 @@ INTERNAL {
>  	roc_sso_hwgrp_qos_config;
>  	roc_sso_hwgrp_release_xaq;
>  	roc_sso_hwgrp_set_priority;
> +	roc_sso_hwgrp_stats_get;
>  	roc_sso_hws_base_get;
>  	roc_sso_hws_link;
> +	roc_sso_hws_stats_get;
>  	roc_sso_hws_unlink;
>  	roc_sso_ns_to_gw;
>  	roc_sso_rsrc_fini;
> diff --git a/drivers/event/cnxk/cnxk_eventdev.h b/drivers/event/cnxk/cnxk_eventdev.h
> index 9af04bc3d..abe36f21f 100644
> --- a/drivers/event/cnxk/cnxk_eventdev.h
> +++ b/drivers/event/cnxk/cnxk_eventdev.h
> @@ -214,6 +214,21 @@ int cnxk_sso_close(struct rte_eventdev *event_dev, cnxk_sso_unlink_t unlink_fn);
>  int cnxk_sso_selftest(const char *dev_name);
>  void cnxk_sso_dump(struct rte_eventdev *event_dev, FILE *f);
>  
> +/* Stats API. */
> +int cnxk_sso_xstats_get_names(const struct rte_eventdev *event_dev,
> +			      enum rte_event_dev_xstats_mode mode,
> +			      uint8_t queue_port_id,
> +			      struct rte_event_dev_xstats_name *xstats_names,
> +			      unsigned int *ids, unsigned int size);
> +int cnxk_sso_xstats_get(const struct rte_eventdev *event_dev,
> +			enum rte_event_dev_xstats_mode mode,
> +			uint8_t queue_port_id, const unsigned int ids[],
> +			uint64_t values[], unsigned int n);
> +int cnxk_sso_xstats_reset(struct rte_eventdev *event_dev,
> +			  enum rte_event_dev_xstats_mode mode,
> +			  int16_t queue_port_id, const uint32_t ids[],
> +			  uint32_t n);
> +
>  /* CN9K */
>  void cn9k_sso_set_rsrc(void *arg);
>  
> diff --git a/drivers/event/cnxk/cnxk_eventdev_stats.c b/drivers/event/cnxk/cnxk_eventdev_stats.c
> new file mode 100644
> index 000000000..a3b548f46
> --- /dev/null
> +++ b/drivers/event/cnxk/cnxk_eventdev_stats.c
> @@ -0,0 +1,289 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(C) 2021 Marvell.
> + */
> +
> +#include "cnxk_eventdev.h"
> +
> +struct cnxk_sso_xstats_name {
> +	const char name[RTE_EVENT_DEV_XSTATS_NAME_SIZE];
> +	const size_t offset;
> +	const uint64_t mask;
> +	const uint8_t shift;
> +	uint64_t reset_snap[CNXK_SSO_MAX_HWGRP];
> +};
> +
> +static struct cnxk_sso_xstats_name sso_hws_xstats[] = {
> +	{
> +		"last_grp_serviced",
> +		offsetof(struct roc_sso_hws_stats, arbitration),
> +		0x3FF,
> +		0,
> +		{0},
> +	},
> +	{
> +		"affinity_arbitration_credits",
> +		offsetof(struct roc_sso_hws_stats, arbitration),
> +		0xF,
> +		16,
> +		{0},
> +	},
> +};
> +
> +static struct cnxk_sso_xstats_name sso_hwgrp_xstats[] = {
> +	{
> +		"wrk_sched",
> +		offsetof(struct roc_sso_hwgrp_stats, ws_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"xaq_dram",
> +		offsetof(struct roc_sso_hwgrp_stats, ext_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"add_wrk",
> +		offsetof(struct roc_sso_hwgrp_stats, wa_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"tag_switch_req",
> +		offsetof(struct roc_sso_hwgrp_stats, ts_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"desched_req",
> +		offsetof(struct roc_sso_hwgrp_stats, ds_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"desched_wrk",
> +		offsetof(struct roc_sso_hwgrp_stats, dq_pc),
> +		~0x0,
> +		0,
> +		{0},
> +	},
> +	{
> +		"xaq_cached",
> +		offsetof(struct roc_sso_hwgrp_stats, aw_status),
> +		0x3,
> +		0,
> +		{0},
> +	},
> +	{
> +		"work_inflight",
> +		offsetof(struct roc_sso_hwgrp_stats, aw_status),
> +		0x3F,
> +		16,
> +		{0},
> +	},
> +	{
> +		"inuse_pages",
> +		offsetof(struct roc_sso_hwgrp_stats, page_cnt),
> +		0xFFFFFFFF,
> +		0,
> +		{0},
> +	},
> +};
> +
> +#define CNXK_SSO_NUM_HWS_XSTATS RTE_DIM(sso_hws_xstats)
> +#define CNXK_SSO_NUM_GRP_XSTATS RTE_DIM(sso_hwgrp_xstats)
> +
> +#define CNXK_SSO_NUM_XSTATS (CNXK_SSO_NUM_HWS_XSTATS + CNXK_SSO_NUM_GRP_XSTATS)
> +
> +int
> +cnxk_sso_xstats_get(const struct rte_eventdev *event_dev,
> +		    enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id,
> +		    const unsigned int ids[], uint64_t values[], unsigned int n)
> +{
> +	struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
> +	struct roc_sso_hwgrp_stats hwgrp_stats;
> +	struct cnxk_sso_xstats_name *xstats;
> +	struct cnxk_sso_xstats_name *xstat;
> +	struct roc_sso_hws_stats hws_stats;
> +	uint32_t xstats_mode_count = 0;
> +	uint32_t start_offset = 0;
> +	unsigned int i;
> +	uint64_t value;
> +	void *rsp;
> +	int rc;
> +
> +	switch (mode) {
> +	case RTE_EVENT_DEV_XSTATS_DEVICE:
> +		return 0;
> +	case RTE_EVENT_DEV_XSTATS_PORT:
> +		if (queue_port_id >= (signed int)dev->nb_event_ports)
> +			goto invalid_value;
> +
> +		xstats_mode_count = CNXK_SSO_NUM_HWS_XSTATS;
> +		xstats = sso_hws_xstats;
> +
> +		rc = roc_sso_hws_stats_get(&dev->sso, queue_port_id,
> +					   &hws_stats);
> +		if (rc < 0)
> +			goto invalid_value;
> +		rsp = &hws_stats;
> +		break;
> +	case RTE_EVENT_DEV_XSTATS_QUEUE:
> +		if (queue_port_id >= (signed int)dev->nb_event_queues)
> +			goto invalid_value;
> +
> +		xstats_mode_count = CNXK_SSO_NUM_GRP_XSTATS;
> +		start_offset = CNXK_SSO_NUM_HWS_XSTATS;
> +		xstats = sso_hwgrp_xstats;
> +
> +		rc = roc_sso_hwgrp_stats_get(&dev->sso, queue_port_id,
> +					     &hwgrp_stats);
> +		if (rc < 0)
> +			goto invalid_value;
> +		rsp = &hwgrp_stats;
> +
> +		break;
> +	default:
> +		plt_err("Invalid mode received");
> +		goto invalid_value;
> +	};
> +
> +	for (i = 0; i < n && i < xstats_mode_count; i++) {
> +		xstat = &xstats[ids[i] - start_offset];
> +		value = *(uint64_t *)((char *)rsp + xstat->offset);
> +		value = (value >> xstat->shift) & xstat->mask;
> +
> +		values[i] = value;
> +		values[i] -= xstat->reset_snap[queue_port_id];
> +	}
> +
> +	return i;
> +invalid_value:
> +	return -EINVAL;
> +}
> +
> +int
> +cnxk_sso_xstats_reset(struct rte_eventdev *event_dev,
> +		      enum rte_event_dev_xstats_mode mode,
> +		      int16_t queue_port_id, const uint32_t ids[], uint32_t n)
> +{
> +	struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
> +	struct roc_sso_hwgrp_stats hwgrp_stats;
> +	struct cnxk_sso_xstats_name *xstats;
> +	struct cnxk_sso_xstats_name *xstat;
> +	struct roc_sso_hws_stats hws_stats;
> +	uint32_t xstats_mode_count = 0;
> +	uint32_t start_offset = 0;
> +	unsigned int i;
> +	uint64_t value;
> +	void *rsp;
> +	int rc;
> +
> +	switch (mode) {
> +	case RTE_EVENT_DEV_XSTATS_DEVICE:
> +		return 0;
> +	case RTE_EVENT_DEV_XSTATS_PORT:
> +		if (queue_port_id >= (signed int)dev->nb_event_ports)
> +			goto invalid_value;
> +
> +		xstats_mode_count = CNXK_SSO_NUM_HWS_XSTATS;
> +		xstats = sso_hws_xstats;
> +		rc = roc_sso_hws_stats_get(&dev->sso, queue_port_id,
> +					   &hws_stats);
> +		if (rc < 0)
> +			goto invalid_value;
> +		rsp = &hws_stats;
> +		break;
> +	case RTE_EVENT_DEV_XSTATS_QUEUE:
> +		if (queue_port_id >= (signed int)dev->nb_event_queues)
> +			goto invalid_value;
> +
> +		xstats_mode_count = CNXK_SSO_NUM_GRP_XSTATS;
> +		start_offset = CNXK_SSO_NUM_HWS_XSTATS;
> +		xstats = sso_hwgrp_xstats;
> +
> +		rc = roc_sso_hwgrp_stats_get(&dev->sso, queue_port_id,
> +					     &hwgrp_stats);
> +		if (rc < 0)
> +			goto invalid_value;
> +		rsp = &hwgrp_stats;
> +		break;
> +	default:
> +		plt_err("Invalid mode received");
> +		goto invalid_value;
> +	};
> +
> +	for (i = 0; i < n && i < xstats_mode_count; i++) {
> +		xstat = &xstats[ids[i] - start_offset];
> +		value = *(uint64_t *)((char *)rsp + xstat->offset);
> +		value = (value >> xstat->shift) & xstat->mask;
> +
> +		xstat->reset_snap[queue_port_id] = value;
> +	}
> +	return i;
> +invalid_value:
> +	return -EINVAL;
> +}
> +
> +int
> +cnxk_sso_xstats_get_names(const struct rte_eventdev *event_dev,
> +			  enum rte_event_dev_xstats_mode mode,
> +			  uint8_t queue_port_id,
> +			  struct rte_event_dev_xstats_name *xstats_names,
> +			  unsigned int *ids, unsigned int size)
> +{
> +	struct rte_event_dev_xstats_name xstats_names_copy[CNXK_SSO_NUM_XSTATS];
> +	struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
> +	uint32_t xstats_mode_count = 0;
> +	uint32_t start_offset = 0;
> +	unsigned int xidx = 0;
> +	unsigned int i;
> +
> +	for (i = 0; i < CNXK_SSO_NUM_HWS_XSTATS; i++) {
> +		snprintf(xstats_names_copy[i].name,
> +			 sizeof(xstats_names_copy[i].name), "%s",
> +			 sso_hws_xstats[i].name);
> +	}
> +
> +	for (; i < CNXK_SSO_NUM_XSTATS; i++) {
> +		snprintf(xstats_names_copy[i].name,
> +			 sizeof(xstats_names_copy[i].name), "%s",
> +			 sso_hwgrp_xstats[i - CNXK_SSO_NUM_HWS_XSTATS].name);
> +	}
> +
> +	switch (mode) {
> +	case RTE_EVENT_DEV_XSTATS_DEVICE:
> +		break;
> +	case RTE_EVENT_DEV_XSTATS_PORT:
> +		if (queue_port_id >= (signed int)dev->nb_event_ports)
> +			break;
> +		xstats_mode_count = CNXK_SSO_NUM_HWS_XSTATS;
> +		break;
> +	case RTE_EVENT_DEV_XSTATS_QUEUE:
> +		if (queue_port_id >= (signed int)dev->nb_event_queues)
> +			break;
> +		xstats_mode_count = CNXK_SSO_NUM_GRP_XSTATS;
> +		start_offset = CNXK_SSO_NUM_HWS_XSTATS;
> +		break;
> +	default:
> +		plt_err("Invalid mode received");
> +		return -EINVAL;
> +	};
> +
> +	if (xstats_mode_count > size || !ids || !xstats_names)
> +		return xstats_mode_count;
> +
> +	for (i = 0; i < xstats_mode_count; i++) {
> +		xidx = i + start_offset;
> +		strncpy(xstats_names[i].name, xstats_names_copy[xidx].name,
> +			sizeof(xstats_names[i].name));
> +		ids[i] = xidx;
> +	}
> +
> +	return i;
> +}
> diff --git a/drivers/event/cnxk/meson.build b/drivers/event/cnxk/meson.build
> index e37ea3478..5b215b73f 100644
> --- a/drivers/event/cnxk/meson.build
> +++ b/drivers/event/cnxk/meson.build
> @@ -13,7 +13,8 @@ sources = files('cn10k_worker.c',
>                  'cn9k_worker.c',
>                  'cn9k_eventdev.c',
>                  'cnxk_eventdev.c',
> -                'cnxk_eventdev_selftest.c'
> +                'cnxk_eventdev_selftest.c',
> +                'cnxk_eventdev_stats.c',
>                  )
>  
>  deps += ['bus_pci', 'common_cnxk']
>