RE: [PATCH v5 5/5] raw/afu_mf: add HE-HSSI AFU driver

["Zhang, Tianfei" <tianfei.zhang@intel.com>]

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Friday, May 27, 2022 1:37 PM
> To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>
> Subject: [PATCH v5 5/5] raw/afu_mf: add HE-HSSI AFU driver
> 
> HE-HSSI is one of the host exerciser modules in OFS FPGA, which is used to test
> HSSI (High Speed Serial Interface).
> This driver initialize the module and report test result.
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> ---
>  drivers/raw/afu_mf/afu_mf_rawdev.c |   3 +
>  drivers/raw/afu_mf/he_hssi.c       | 369

The filename change to "afu_pmd_he_hssi.c" is better?

> +++++++++++++++++++++++++++++++++++++
>  drivers/raw/afu_mf/he_hssi.h       | 102 ++++++++++
>  drivers/raw/afu_mf/meson.build     |   3 +-
>  drivers/raw/afu_mf/rte_pmd_afu.h   |  16 ++
>  5 files changed, 492 insertions(+), 1 deletion(-)  create mode 100644
> drivers/raw/afu_mf/he_hssi.c  create mode 100644
> drivers/raw/afu_mf/he_hssi.h
> 
> diff --git a/drivers/raw/afu_mf/afu_mf_rawdev.c
> b/drivers/raw/afu_mf/afu_mf_rawdev.c
> index a56f60e..f24c748 100644
> --- a/drivers/raw/afu_mf/afu_mf_rawdev.c
> +++ b/drivers/raw/afu_mf/afu_mf_rawdev.c
> @@ -22,6 +22,7 @@
>  #include "n3000_afu.h"
>  #include "he_lbk.h"
>  #include "he_mem.h"
> +#include "he_hssi.h"
> 
>  #define AFU_MF_PMD_RAWDEV_NAME rawdev_afu_mf
> 
> @@ -30,6 +31,7 @@
>  	{ HE_LBK_UUID_L, HE_LBK_UUID_H },
>  	{ HE_MEM_LBK_UUID_L, HE_MEM_LBK_UUID_H },
>  	{ HE_MEM_TG_UUID_L, HE_MEM_TG_UUID_H },
> +	{ HE_HSSI_UUID_L, HE_HSSI_UUID_H },
>  	{ 0, 0 /* sentinel */ }
>  };
> 
> @@ -38,6 +40,7 @@
>  	&he_lbk_drv,
>  	&he_mem_lbk_drv,
>  	&he_mem_tg_drv,
> +	&he_hssi_drv,
>  	NULL
>  };
> 
> diff --git a/drivers/raw/afu_mf/he_hssi.c b/drivers/raw/afu_mf/he_hssi.c new
> file mode 100644 index 0000000..bedafbd
> --- /dev/null
> +++ b/drivers/raw/afu_mf/he_hssi.c
> @@ -0,0 +1,369 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#include <errno.h>
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <poll.h>
> +#include <sys/eventfd.h>
> +#include <sys/ioctl.h>
> +
> +#include <rte_eal.h>
> +#include <rte_malloc.h>
> +#include <rte_memcpy.h>
> +#include <rte_io.h>
> +#include <rte_vfio.h>
> +#include <rte_bus_pci.h>
> +#include <rte_bus_ifpga.h>
> +#include <rte_rawdev.h>
> +
> +#include "afu_mf_rawdev.h"
> +#include "he_hssi.h"
> +
> +static int he_hssi_indirect_write(struct he_hssi_ctx *ctx, uint32_t addr,
> +	uint32_t value)
> +{
> +	struct traffic_ctrl_cmd cmd;
> +	struct traffic_ctrl_data data;
> +	uint32_t i = 0;
> +
> +	AFU_MF_PMD_DEBUG("Indirect write 0x%x, value 0x%08x", addr,
> value);
> +
> +	if (!ctx)
> +		return -EINVAL;
> +
> +	data.write_data = value;
> +	rte_write64(data.csr, ctx->addr + TRAFFIC_CTRL_DATA);
> +
> +	cmd.csr = 0;
> +	cmd.write_cmd = 1;
> +	cmd.afu_cmd_addr = addr;
> +	rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	i = 0;
> +	cmd.csr = 0;
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		cmd.ack_trans = 1;
> +		rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (!cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_indirect_read(struct he_hssi_ctx *ctx, uint32_t addr,
> +	uint32_t *value)
> +{
> +	struct traffic_ctrl_cmd cmd;
> +	struct traffic_ctrl_data data;
> +	uint32_t i = 0;
> +
> +	if (!ctx)
> +		return -EINVAL;
> +
> +	cmd.csr = 0;
> +	cmd.read_cmd = 1;
> +	cmd.afu_cmd_addr = addr;
> +	rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (cmd.ack_trans) {
> +			data.csr = rte_read64(ctx->addr +
> TRAFFIC_CTRL_DATA);
> +			*value = data.read_data;
> +			break;
> +		}
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	i = 0;
> +	cmd.csr = 0;
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		cmd.ack_trans = 1;
> +		rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (!cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	AFU_MF_PMD_DEBUG("Indirect read 0x%x, value 0x%08x", addr,
> *value);
> +	return 0;
> +}
> +
> +static void he_hssi_report(struct he_hssi_ctx *ctx) {
> +	uint32_t val = 0;
> +	uint64_t v64 = 0;
> +	int ret = 0;
> +
> +	ret = he_hssi_indirect_read(ctx, TM_PKT_GOOD, &val);
> +	if (ret)
> +		return;
> +	printf("Number of good packets received: %u\n", val);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_PKT_BAD, &val);
> +	if (ret)
> +		return;
> +	printf("Number of bad packets received: %u\n", val);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT1, &val);
> +	if (ret)
> +		return;
> +	v64 = val;
> +	ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT0, &val);
> +	if (ret)
> +		return;
> +	v64 = (v64 << 32) | val;
> +	printf("Number of bytes received: %"PRIu64"\n", v64);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_AVST_RX_ERR, &val);
> +	if (ret)
> +		return;
> +	if (val & ERR_VALID) {
> +		printf("AVST rx error:");
> +		if (val & OVERFLOW_ERR)
> +			printf(" overflow");
> +		if (val & LENGTH_ERR)
> +			printf(" length");
> +		if (val & OVERSIZE_ERR)
> +			printf(" oversize");
> +		if (val & UNDERSIZE_ERR)
> +			printf(" undersize");
> +		if (val & MAC_CRC_ERR)
> +			printf(" crc");
> +		if (val & PHY_ERR)
> +			printf(" phy");
> +		printf("\n");
> +	}
> +
> +	ret = he_hssi_indirect_read(ctx, LOOPBACK_FIFO_STATUS, &val);
> +	if (ret)
> +		return;
> +	if (val & (ALMOST_EMPTY | ALMOST_FULL)) {
> +		printf("FIFO status:");
> +		if (val & ALMOST_EMPTY)
> +			printf(" almost empty");
> +		if (val & ALMOST_FULL)
> +			printf(" almost full");
> +		printf("\n");
> +	}
> +}
> +
> +static int he_hssi_test(struct afu_mf_rawdev *dev) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +	struct traffic_ctrl_ch_sel sel;
> +	uint32_t val = 0;
> +	uint32_t i = 0;
> +	int ret = 0;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	cfg = &priv->he_hssi_cfg;
> +	ctx = &priv->he_hssi_ctx;
> +
> +	ret = he_hssi_indirect_write(ctx, TG_STOP_XFR, 0);
> +	if (ret)
> +		return ret;
> +
> +	sel.channel_sel = cfg->port;
> +	rte_write64(sel.csr, ctx->addr + TRAFFIC_CTRL_CH_SEL);
> +
> +	if (cfg->he_loopback >= 0) {
> +		val = cfg->he_loopback ? 1 : 0;
> +		AFU_MF_PMD_INFO("%s HE loopback on port %u",
> +			val ? "Enable" : "Disable", cfg->port);
> +		return he_hssi_indirect_write(ctx, LOOPBACK_EN, val);
> +	}
> +
> +	ret = he_hssi_indirect_write(ctx, TG_NUM_PKT, cfg->num_packets);
> +	if (ret)
> +		return ret;
> +
> +	ret = he_hssi_indirect_write(ctx, TG_PKT_LEN, cfg->packet_length);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->src_addr & 0xffffffff;
> +	ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_L, val);
> +	if (ret)
> +		return ret;
> +	val = (cfg->src_addr >> 32) & 0xffff;
> +	ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_H, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->dest_addr & 0xffffffff;
> +	ret = he_hssi_indirect_write(ctx, TG_DST_MAC_L, val);
> +	if (ret)
> +		return ret;
> +	val = (cfg->dest_addr >> 32) & 0xffff;
> +	ret = he_hssi_indirect_write(ctx, TG_DST_MAC_H, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->random_length ? 1 : 0;
> +	ret = he_hssi_indirect_write(ctx, TG_PKT_LEN_TYPE, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->random_payload ? 1 : 0;
> +	ret = he_hssi_indirect_write(ctx, TG_DATA_PATTERN, val);
> +	if (ret)
> +		return ret;
> +
> +	for (i = 0; i < 3; i++) {

What meaning of "3" ?

> +		ret = he_hssi_indirect_write(ctx, TG_RANDOM_SEED(i),
> +			cfg->rnd_seed[i]);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	ret = he_hssi_indirect_write(ctx, TG_START_XFR, 1);
> +	if (ret)
> +		return ret;
> +
> +	while (i++ < cfg->timeout) {
> +		ret = he_hssi_indirect_read(ctx, TG_PKT_XFRD, &val);
> +		if (ret)
> +			break;
> +		if (val == cfg->num_packets)
> +			break;
> +		sleep(1);
> +	}
> +
> +	he_hssi_report(ctx);
> +
> +	return ret;
> +}
> +
> +static int he_hssi_init(struct afu_mf_rawdev *dev) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv) {
> +		priv = rte_zmalloc(NULL, sizeof(struct he_hssi_priv), 0);
> +		if (!priv)
> +			return -ENOMEM;
> +		dev->priv = priv;
> +	}
> +
> +	ctx = &priv->he_hssi_ctx;
> +	ctx->addr = (uint8_t *)dev->addr;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_config(struct afu_mf_rawdev *dev, void *config,
> +	size_t config_size)
> +{
> +	struct he_hssi_priv *priv = NULL;
> +	struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> +
> +	if (!dev || !config || !config_size)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	if (config_size != sizeof(struct rte_pmd_afu_he_hssi_cfg))
> +		return -EINVAL;
> +
> +	cfg = (struct rte_pmd_afu_he_hssi_cfg *)config;
> +	if (cfg->port >= NUM_HE_HSSI_PORTS)
> +		return -EINVAL;
> +
> +	rte_memcpy(&priv->he_hssi_cfg, cfg, sizeof(priv->he_hssi_cfg));
> +
> +	return 0;
> +}
> +
> +static int he_hssi_close(struct afu_mf_rawdev *dev) {
> +	if (!dev)
> +		return -EINVAL;
> +
> +	rte_free(dev->priv);
> +	dev->priv = NULL;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_dump(struct afu_mf_rawdev *dev, FILE *f) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	if (!f)
> +		f = stdout;
> +
> +	ctx = &priv->he_hssi_ctx;
> +
> +	fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
> +
> +	return 0;
> +}
> +
> +static struct afu_mf_ops he_hssi_ops = {
> +	.init = he_hssi_init,
> +	.config = he_hssi_config,
> +	.start = NULL,
> +	.stop = NULL,
> +	.test = he_hssi_test,
> +	.close = he_hssi_close,
> +	.dump = he_hssi_dump,
> +	.reset = NULL
> +};
> +
> +struct afu_mf_drv he_hssi_drv = {
> +	.uuid = { HE_HSSI_UUID_L, HE_HSSI_UUID_H },
> +	.ops = &he_hssi_ops
> +};
> diff --git a/drivers/raw/afu_mf/he_hssi.h b/drivers/raw/afu_mf/he_hssi.h new
> file mode 100644 index 0000000..f8b9623
> --- /dev/null
> +++ b/drivers/raw/afu_mf/he_hssi.h
> @@ -0,0 +1,102 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#ifndef _HE_HSSI_H_
> +#define _HE_HSSI_H_
> +
> +#include "afu_mf_rawdev.h"
> +#include "rte_pmd_afu.h"
> +
> +#define HE_HSSI_UUID_L    0xbb370242ac130002
> +#define HE_HSSI_UUID_H    0x823c334c98bf11ea
> +#define NUM_HE_HSSI_PORTS 8
> +
> +extern struct afu_mf_drv he_hssi_drv;
> +
> +/* HE-HSSI registers definition */
> +#define TRAFFIC_CTRL_CMD    0x30
> +#define TRAFFIC_CTRL_DATA   0x38
> +#define TRAFFIC_CTRL_CH_SEL 0x40
> +#define AFU_SCRATCHPAD      0x48
> +
> +#define TG_NUM_PKT        0x3c00
> +#define TG_PKT_LEN_TYPE   0x3c01
> +#define TG_DATA_PATTERN   0x3c02
> +#define TG_START_XFR      0x3c03
> +#define TG_STOP_XFR       0x3c04
> +#define TG_SRC_MAC_L      0x3c05
> +#define TG_SRC_MAC_H      0x3c06
> +#define TG_DST_MAC_L      0x3c07
> +#define TG_DST_MAC_H      0x3c08
> +#define TG_PKT_XFRD       0x3c09
> +#define TG_RANDOM_SEED(n) (0x3c0a + (n))
> +#define TG_PKT_LEN        0x3c0d
> +
> +#define TM_NUM_PKT        0x3d00
> +#define TM_PKT_GOOD       0x3d01
> +#define TM_PKT_BAD        0x3d02
> +#define TM_BYTE_CNT0      0x3d03
> +#define TM_BYTE_CNT1      0x3d04
> +#define TM_AVST_RX_ERR    0x3d07
> +#define   OVERFLOW_ERR    (1 << 9)
> +#define   LENGTH_ERR      (1 << 8)
> +#define   OVERSIZE_ERR    (1 << 7)
> +#define   UNDERSIZE_ERR   (1 << 6)
> +#define   MAC_CRC_ERR     (1 << 5)
> +#define   PHY_ERR         (1 << 4)
> +#define   ERR_VALID       (1 << 3)
> +
> +#define LOOPBACK_EN          0x3e00
> +#define LOOPBACK_FIFO_STATUS 0x3e01
> +#define   ALMOST_EMPTY    (1 << 1)
> +#define   ALMOST_FULL     (1 << 0)
> +
> +#define MAILBOX_TIMEOUT_MS       100
> +#define MAILBOX_POLL_INTERVAL_MS 10
> +
> +struct traffic_ctrl_cmd {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t read_cmd:1;
> +			uint32_t write_cmd:1;
> +			uint32_t ack_trans:1;
> +			uint32_t rsvd1:29;
> +			uint32_t afu_cmd_addr:16;
> +			uint32_t rsvd2:16;
> +		};
> +	};
> +};
> +
> +struct traffic_ctrl_data {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t read_data;
> +			uint32_t write_data;
> +		};
> +	};
> +};
> +
> +struct traffic_ctrl_ch_sel {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t channel_sel:3;
> +			uint32_t rsvd1:29;
> +			uint32_t rsvd2;
> +		};
> +	};
> +};
> +
> +struct he_hssi_ctx {
> +	uint8_t *addr;
> +};
> +
> +struct he_hssi_priv {
> +	struct rte_pmd_afu_he_hssi_cfg he_hssi_cfg;
> +	struct he_hssi_ctx he_hssi_ctx;
> +};
> +
> +#endif /* _HE_HSSI_H_ */
> diff --git a/drivers/raw/afu_mf/meson.build b/drivers/raw/afu_mf/meson.build
> index b53a31b..f304bc8 100644
> --- a/drivers/raw/afu_mf/meson.build
> +++ b/drivers/raw/afu_mf/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright 2022 Intel Corporation
> 
>  deps += ['rawdev', 'bus_pci', 'bus_ifpga'] -sources = files('afu_mf_rawdev.c',
> 'n3000_afu.c', 'he_lbk.c', 'he_mem.c')
> +sources = files('afu_mf_rawdev.c', 'n3000_afu.c', 'he_lbk.c', 'he_mem.c',
> +	'he_hssi.c')
> 
>  headers = files('rte_pmd_afu.h')
> diff --git a/drivers/raw/afu_mf/rte_pmd_afu.h
> b/drivers/raw/afu_mf/rte_pmd_afu.h
> index 2f92f7e..89d866a 100644
> --- a/drivers/raw/afu_mf/rte_pmd_afu.h
> +++ b/drivers/raw/afu_mf/rte_pmd_afu.h
> @@ -111,6 +111,22 @@ struct rte_pmd_afu_he_mem_tg_cfg {
>  	uint32_t channel_mask;   /* mask of traffic generator channel */
>  };
> 
> +/**
> + * HE-HSSI AFU configuration data structure.
> + */
> +struct rte_pmd_afu_he_hssi_cfg {
> +	uint32_t port;
> +	uint32_t timeout;
> +	uint32_t num_packets;
> +	uint32_t random_length;
> +	uint32_t packet_length;
> +	uint32_t random_payload;
> +	uint32_t rnd_seed[3];
> +	uint64_t src_addr;
> +	uint64_t dest_addr;
> +	int he_loopback;
> +};
> +
>  #ifdef __cplusplus
>  }
>  #endif
> --
> 1.8.3.1