RE: [PATCH v7 5/5] raw/ifpga: add HE-HSSI AFU driver

["Xu, Rosen" <rosen.xu@intel.com>]

Hi Wei,

> -----Original Message-----
> From: Huang, Wei <wei.huang@intel.com>
> Sent: Thursday, June 09, 2022 15:37
> To: dev@dpdk.org; thomas@monjalon.net; nipun.gupta@nxp.com;
> hemant.agrawal@nxp.com
> Cc: stable@dpdk.org; Xu, Rosen <rosen.xu@intel.com>; Zhang, Tianfei
> <tianfei.zhang@intel.com>; Zhang, Qi Z <qi.z.zhang@intel.com>; Huang, Wei
> <wei.huang@intel.com>
> Subject: [PATCH v7 5/5] raw/ifpga: add HE-HSSI AFU driver
> 
> HE-HSSI is one of the host exerciser modules in OFS FPGA, which is used to
> test HSSI (High Speed Serial Interface).
> This driver initialize the module and report test result.
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> ---
> v2: move source files to ifpga and rename, refine code
> ---
>  drivers/raw/ifpga/afu_pmd_he_hssi.c | 371
> ++++++++++++++++++++++++++++++++++++
>  drivers/raw/ifpga/afu_pmd_he_hssi.h | 109 +++++++++++
>  drivers/raw/ifpga/meson.build       |   3 +-
>  drivers/raw/ifpga/rte_pmd_afu.h     |  18 ++
>  4 files changed, 500 insertions(+), 1 deletion(-)  create mode 100644
> drivers/raw/ifpga/afu_pmd_he_hssi.c
>  create mode 100644 drivers/raw/ifpga/afu_pmd_he_hssi.h
> 
> diff --git a/drivers/raw/ifpga/afu_pmd_he_hssi.c
> b/drivers/raw/ifpga/afu_pmd_he_hssi.c
> new file mode 100644
> index 0000000..102de50
> --- /dev/null
> +++ b/drivers/raw/ifpga/afu_pmd_he_hssi.c
> @@ -0,0 +1,371 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#include <errno.h>
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <poll.h>
> +#include <sys/eventfd.h>
> +#include <sys/ioctl.h>
> +
> +#include <rte_eal.h>
> +#include <rte_malloc.h>
> +#include <rte_memcpy.h>
> +#include <rte_io.h>
> +#include <rte_vfio.h>
> +#include <rte_bus_pci.h>
> +#include <rte_bus_ifpga.h>
> +#include <rte_rawdev.h>
> +
> +#include "afu_pmd_core.h"
> +#include "afu_pmd_he_hssi.h"
> +
> +static int he_hssi_indirect_write(struct he_hssi_ctx *ctx, uint32_t addr,
> +	uint32_t value)
> +{
> +	struct traffic_ctrl_cmd cmd;
> +	struct traffic_ctrl_data data;
> +	uint32_t i = 0;
> +
> +	IFPGA_RAWDEV_PMD_DEBUG("Indirect write 0x%x, value 0x%08x",
> addr,
> +value);
> +
> +	if (!ctx)
> +		return -EINVAL;
> +
> +	data.write_data = value;
> +	rte_write64(data.csr, ctx->addr + TRAFFIC_CTRL_DATA);
> +
> +	cmd.csr = 0;
> +	cmd.write_cmd = 1;
> +	cmd.afu_cmd_addr = addr;
> +	rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	i = 0;
> +	cmd.csr = 0;
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		cmd.ack_trans = 1;
> +		rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (!cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_indirect_read(struct he_hssi_ctx *ctx, uint32_t addr,
> +	uint32_t *value)
> +{
> +	struct traffic_ctrl_cmd cmd;
> +	struct traffic_ctrl_data data;
> +	uint32_t i = 0;
> +
> +	if (!ctx)
> +		return -EINVAL;
> +
> +	cmd.csr = 0;
> +	cmd.read_cmd = 1;
> +	cmd.afu_cmd_addr = addr;
> +	rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (cmd.ack_trans) {
> +			data.csr = rte_read64(ctx->addr +
> TRAFFIC_CTRL_DATA);
> +			*value = data.read_data;
> +			break;
> +		}
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	i = 0;
> +	cmd.csr = 0;
> +	while (i < MAILBOX_TIMEOUT_MS) {
> +		cmd.ack_trans = 1;
> +		rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +		rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> +		cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> +		if (!cmd.ack_trans)
> +			break;
> +		i += MAILBOX_POLL_INTERVAL_MS;
> +	}
> +	if (i >= MAILBOX_TIMEOUT_MS)
> +		return -ETIMEDOUT;
> +
> +	IFPGA_RAWDEV_PMD_DEBUG("Indirect read 0x%x, value 0x%08x",
> addr, *value);
> +	return 0;
> +}
> +
> +static void he_hssi_report(struct he_hssi_ctx *ctx) {
> +	uint32_t val = 0;
> +	uint64_t v64 = 0;
> +	int ret = 0;
> +
> +	ret = he_hssi_indirect_read(ctx, TM_PKT_GOOD, &val);
> +	if (ret)
> +		return;
> +	printf("Number of good packets received: %u\n", val);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_PKT_BAD, &val);
> +	if (ret)
> +		return;
> +	printf("Number of bad packets received: %u\n", val);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT1, &val);
> +	if (ret)
> +		return;
> +	v64 = val;
> +	ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT0, &val);
> +	if (ret)
> +		return;
> +	v64 = (v64 << 32) | val;
> +	printf("Number of bytes received: %"PRIu64"\n", v64);
> +
> +	ret = he_hssi_indirect_read(ctx, TM_AVST_RX_ERR, &val);
> +	if (ret)
> +		return;
> +	if (val & ERR_VALID) {
> +		printf("AVST rx error:");
> +		if (val & OVERFLOW_ERR)
> +			printf(" overflow");
> +		if (val & LENGTH_ERR)
> +			printf(" length");
> +		if (val & OVERSIZE_ERR)
> +			printf(" oversize");
> +		if (val & UNDERSIZE_ERR)
> +			printf(" undersize");
> +		if (val & MAC_CRC_ERR)
> +			printf(" crc");
> +		if (val & PHY_ERR)
> +			printf(" phy");
> +		printf("\n");
> +	}
> +
> +	ret = he_hssi_indirect_read(ctx, LOOPBACK_FIFO_STATUS, &val);
> +	if (ret)
> +		return;
> +	if (val & (ALMOST_EMPTY | ALMOST_FULL)) {
> +		printf("FIFO status:");
> +		if (val & ALMOST_EMPTY)
> +			printf(" almost empty");
> +		if (val & ALMOST_FULL)
> +			printf(" almost full");
> +		printf("\n");
> +	}
> +}
> +
> +static int he_hssi_test(struct afu_rawdev *dev) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +	struct traffic_ctrl_ch_sel sel;
> +	uint32_t val = 0;
> +	uint32_t i = 0;
> +	int ret = 0;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	cfg = &priv->he_hssi_cfg;
> +	ctx = &priv->he_hssi_ctx;
> +
> +	ret = he_hssi_indirect_write(ctx, TG_STOP_XFR, 0);
> +	if (ret)
> +		return ret;
> +
> +	sel.channel_sel = cfg->port;
> +	rte_write64(sel.csr, ctx->addr + TRAFFIC_CTRL_CH_SEL);
> +
> +	if (cfg->he_loopback >= 0) {
> +		val = cfg->he_loopback ? 1 : 0;
> +		IFPGA_RAWDEV_PMD_INFO("%s HE loopback on port %u",
> +			val ? "Enable" : "Disable", cfg->port);
> +		return he_hssi_indirect_write(ctx, LOOPBACK_EN, val);
> +	}
> +
> +	ret = he_hssi_indirect_write(ctx, TG_NUM_PKT, cfg->num_packets);
> +	if (ret)
> +		return ret;
> +
> +	ret = he_hssi_indirect_write(ctx, TG_PKT_LEN, cfg->packet_length);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->src_addr & 0xffffffff;
> +	ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_L, val);
> +	if (ret)
> +		return ret;
> +	val = (cfg->src_addr >> 32) & 0xffff;
> +	ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_H, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->dest_addr & 0xffffffff;
> +	ret = he_hssi_indirect_write(ctx, TG_DST_MAC_L, val);
> +	if (ret)
> +		return ret;
> +	val = (cfg->dest_addr >> 32) & 0xffff;
> +	ret = he_hssi_indirect_write(ctx, TG_DST_MAC_H, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->random_length ? 1 : 0;
> +	ret = he_hssi_indirect_write(ctx, TG_PKT_LEN_TYPE, val);
> +	if (ret)
> +		return ret;
> +
> +	val = cfg->random_payload ? 1 : 0;
> +	ret = he_hssi_indirect_write(ctx, TG_DATA_PATTERN, val);
> +	if (ret)
> +		return ret;
> +
> +	for (i = 0; i < TG_NUM_RND_SEEDS; i++) {
> +		ret = he_hssi_indirect_write(ctx, TG_RANDOM_SEED(i),
> +			cfg->rnd_seed[i]);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	ret = he_hssi_indirect_write(ctx, TG_START_XFR, 1);
> +	if (ret)
> +		return ret;
> +
> +	while (i++ < cfg->timeout) {
> +		ret = he_hssi_indirect_read(ctx, TG_PKT_XFRD, &val);
> +		if (ret)
> +			break;
> +		if (val == cfg->num_packets)
> +			break;
> +		sleep(1);
> +	}
> +
> +	he_hssi_report(ctx);
> +
> +	return ret;
> +}
> +
> +static int he_hssi_init(struct afu_rawdev *dev) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv) {
> +		priv = rte_zmalloc(NULL, sizeof(struct he_hssi_priv), 0);
> +		if (!priv)
> +			return -ENOMEM;
> +		dev->priv = priv;
> +	}
> +
> +	ctx = &priv->he_hssi_ctx;
> +	ctx->addr = (uint8_t *)dev->addr;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_config(struct afu_rawdev *dev, void *config,
> +	size_t config_size)
> +{
> +	struct he_hssi_priv *priv = NULL;
> +	struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> +
> +	if (!dev || !config || !config_size)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	if (config_size != sizeof(struct rte_pmd_afu_he_hssi_cfg))
> +		return -EINVAL;
> +
> +	cfg = (struct rte_pmd_afu_he_hssi_cfg *)config;
> +	if (cfg->port >= NUM_HE_HSSI_PORTS)
> +		return -EINVAL;
> +
> +	rte_memcpy(&priv->he_hssi_cfg, cfg, sizeof(priv->he_hssi_cfg));
> +
> +	return 0;
> +}
> +
> +static int he_hssi_close(struct afu_rawdev *dev) {
> +	if (!dev)
> +		return -EINVAL;
> +
> +	rte_free(dev->priv);
> +	dev->priv = NULL;
> +
> +	return 0;
> +}
> +
> +static int he_hssi_dump(struct afu_rawdev *dev, FILE *f) {
> +	struct he_hssi_priv *priv = NULL;
> +	struct he_hssi_ctx *ctx = NULL;
> +
> +	if (!dev)
> +		return -EINVAL;
> +
> +	priv = (struct he_hssi_priv *)dev->priv;
> +	if (!priv)
> +		return -ENOENT;
> +
> +	if (!f)
> +		f = stdout;
> +
> +	ctx = &priv->he_hssi_ctx;
> +
> +	fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
> +
> +	return 0;
> +}
> +
> +static struct afu_ops he_hssi_ops = {
> +	.init = he_hssi_init,
> +	.config = he_hssi_config,
> +	.start = NULL,
> +	.stop = NULL,
> +	.test = he_hssi_test,
> +	.close = he_hssi_close,
> +	.dump = he_hssi_dump,
> +	.reset = NULL
> +};
> +
> +struct afu_rawdev_drv he_hssi_drv = {
> +	.uuid = { HE_HSSI_UUID_L, HE_HSSI_UUID_H },
> +	.ops = &he_hssi_ops
> +};
> +
> +AFU_PMD_REGISTER(he_hssi_drv);
> diff --git a/drivers/raw/ifpga/afu_pmd_he_hssi.h
> b/drivers/raw/ifpga/afu_pmd_he_hssi.h
> new file mode 100644
> index 0000000..38b433c
> --- /dev/null
> +++ b/drivers/raw/ifpga/afu_pmd_he_hssi.h
> @@ -0,0 +1,109 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#ifndef _AFU_PMD_HE_HSSI_H_
> +#define _AFU_PMD_HE_HSSI_H_
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include "afu_pmd_core.h"
> +#include "rte_pmd_afu.h"
> +
> +#define HE_HSSI_UUID_L    0xbb370242ac130002
> +#define HE_HSSI_UUID_H    0x823c334c98bf11ea
> +#define NUM_HE_HSSI_PORTS 8
> +
> +/* HE-HSSI registers definition */
> +#define TRAFFIC_CTRL_CMD    0x30
> +#define TRAFFIC_CTRL_DATA   0x38
> +#define TRAFFIC_CTRL_CH_SEL 0x40
> +#define AFU_SCRATCHPAD      0x48
> +
> +#define TG_NUM_PKT        0x3c00
> +#define TG_PKT_LEN_TYPE   0x3c01
> +#define TG_DATA_PATTERN   0x3c02
> +#define TG_START_XFR      0x3c03
> +#define TG_STOP_XFR       0x3c04
> +#define TG_SRC_MAC_L      0x3c05
> +#define TG_SRC_MAC_H      0x3c06
> +#define TG_DST_MAC_L      0x3c07
> +#define TG_DST_MAC_H      0x3c08
> +#define TG_PKT_XFRD       0x3c09
> +#define TG_NUM_RND_SEEDS  3
> +#define TG_RANDOM_SEED(n) (0x3c0a + (n))
> +#define TG_PKT_LEN        0x3c0d
> +
> +#define TM_NUM_PKT        0x3d00
> +#define TM_PKT_GOOD       0x3d01
> +#define TM_PKT_BAD        0x3d02
> +#define TM_BYTE_CNT0      0x3d03
> +#define TM_BYTE_CNT1      0x3d04
> +#define TM_AVST_RX_ERR    0x3d07
> +#define   OVERFLOW_ERR    (1 << 9)
> +#define   LENGTH_ERR      (1 << 8)
> +#define   OVERSIZE_ERR    (1 << 7)
> +#define   UNDERSIZE_ERR   (1 << 6)
> +#define   MAC_CRC_ERR     (1 << 5)
> +#define   PHY_ERR         (1 << 4)
> +#define   ERR_VALID       (1 << 3)
> +
> +#define LOOPBACK_EN          0x3e00
> +#define LOOPBACK_FIFO_STATUS 0x3e01
> +#define   ALMOST_EMPTY    (1 << 1)
> +#define   ALMOST_FULL     (1 << 0)
> +
> +#define MAILBOX_TIMEOUT_MS       100
> +#define MAILBOX_POLL_INTERVAL_MS 10
> +
> +struct traffic_ctrl_cmd {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t read_cmd:1;
> +			uint32_t write_cmd:1;
> +			uint32_t ack_trans:1;
> +			uint32_t rsvd1:29;
> +			uint32_t afu_cmd_addr:16;
> +			uint32_t rsvd2:16;
> +		};
> +	};
> +};
> +
> +struct traffic_ctrl_data {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t read_data;
> +			uint32_t write_data;
> +		};
> +	};
> +};
> +
> +struct traffic_ctrl_ch_sel {
> +	union {
> +		uint64_t csr;
> +		struct {
> +			uint32_t channel_sel:3;
> +			uint32_t rsvd1:29;
> +			uint32_t rsvd2;
> +		};
> +	};
> +};
> +
> +struct he_hssi_ctx {
> +	uint8_t *addr;
> +};
> +
> +struct he_hssi_priv {
> +	struct rte_pmd_afu_he_hssi_cfg he_hssi_cfg;
> +	struct he_hssi_ctx he_hssi_ctx;
> +};
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _AFU_PMD_HE_HSSI_H_ */
> diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
> index dc6941d..cc30dc8 100644
> --- a/drivers/raw/ifpga/meson.build
> +++ b/drivers/raw/ifpga/meson.build
> @@ -14,7 +14,8 @@ deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
>      'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke']
> 
>  sources = files('ifpga_rawdev.c', 'rte_pmd_ifpga.c', 'afu_pmd_core.c',
> -    'afu_pmd_n3000.c', 'afu_pmd_he_lpbk.c', 'afu_pmd_he_mem.c')
> +    'afu_pmd_n3000.c', 'afu_pmd_he_lpbk.c', 'afu_pmd_he_mem.c',
> +    'afu_pmd_he_hssi.c')
> 
>  includes += include_directories('base')  includes +=
> include_directories('../../net/ipn3ke')
> diff --git a/drivers/raw/ifpga/rte_pmd_afu.h
> b/drivers/raw/ifpga/rte_pmd_afu.h index 213e854..b1cebbe 100644
> --- a/drivers/raw/ifpga/rte_pmd_afu.h
> +++ b/drivers/raw/ifpga/rte_pmd_afu.h
> @@ -111,6 +111,24 @@ struct rte_pmd_afu_he_mem_tg_cfg {
>  	uint32_t channel_mask;   /* mask of traffic generator channel */
>  };
> 
> +#define NUM_RND_SEEDS  3
> +
> +/**
> + * HE-HSSI AFU configuration data structure.
> + */
> +struct rte_pmd_afu_he_hssi_cfg {
> +	uint32_t port;
> +	uint32_t timeout;
> +	uint32_t num_packets;
> +	uint32_t random_length;
> +	uint32_t packet_length;
> +	uint32_t random_payload;
> +	uint32_t rnd_seed[NUM_RND_SEEDS];
> +	uint64_t src_addr;
> +	uint64_t dest_addr;
> +	int he_loopback;
> +};
> +
>  #ifdef __cplusplus
>  }
>  #endif
> --
> 1.8.3.1

Reviewed-by: Rosen Xu <rosen.xu@intel.com>