[dpdk-dev] [PATCH 04/16] nfp: add nsp support for commands

[Alejandro Lucero <alejandro.lucero@netronome.com>]

NSPU interface declares a buffer controlled by the NFP NSP service
processor. It is possible to send commands to the NSP using the NSPU
and this buffer for data related to the command. A command can imply
buffer read, buffer write, both or none.

Initial command for resetting the firmware is added as well which
does not require the buffer at all.

Commands will allow firmware upload, symbol resolution and ethernet
link configuration. Future commands will allow specific offloads like
flow offloads and eBPF offload.

Signed-off-by: Alejandro Lucero <alejandro.lucero@netronome.com>
---
 drivers/net/nfp/nfp_nspu.c | 181 ++++++++++++++++++++++++++++++++++++++++++++-
 drivers/net/nfp/nfp_nspu.h |   4 +
 2 files changed, 183 insertions(+), 2 deletions(-)

diff --git a/drivers/net/nfp/nfp_nspu.c b/drivers/net/nfp/nfp_nspu.c
index a157915..dbb5305 100644
--- a/drivers/net/nfp/nfp_nspu.c
+++ b/drivers/net/nfp/nfp_nspu.c
@@ -29,14 +29,19 @@
 #define NSP_STATUS              0x00
 #define NSP_COMMAND             0x08
 #define NSP_BUFFER		0x10
-#define NSP_DEFAULT_BUFFER      0x18
-#define NSP_DEFAULT_BUFFER_CFG  0x20
+#define NSP_DEFAULT_BUF         0x18
+#define NSP_DEFAULT_BUF_CFG  0x20
 
 #define NSP_MAGIC                0xab10
 #define NSP_STATUS_MAGIC(x)      (((x) >> 48) & 0xffff)
 #define NSP_STATUS_MAJOR(x)      (int)(((x) >> 44) & 0xf)
 #define NSP_STATUS_MINOR(x)      (int)(((x) >> 32) & 0xfff)
 
+/* NSP commands */
+#define NSP_CMD_RESET			1
+
+#define NSP_BUFFER_CFG_SIZE_MASK	(0xff)
+
 #define NSP_REG_ADDR(d, off, reg) ((uint8_t *)(d)->mem_base + (off) + (reg))
 #define NSP_REG_VAL(p) (*(uint64_t *)(p))
 
@@ -118,12 +123,184 @@
 nfp_nspu_init(nspu_desc_t *desc, int nfp, int pcie_bar, size_t pcie_barsz,
 	      int exp_bar, void *exp_bar_cfg_base, void *exp_bar_mmap)
 {
+	uint64_t offset, buffaddr;
+	uint64_t nsp_reg;
+
 	desc->nfp = nfp;
 	desc->pcie_bar = pcie_bar;
 	desc->exp_bar = exp_bar;
 	desc->barsz = pcie_barsz;
+	desc->windowsz = 1 << (desc->barsz - 3);
 	desc->cfg_base = exp_bar_cfg_base;
 	desc->mem_base = exp_bar_mmap;
 
+	nspu_xlate(desc, NSP_BASE, &offset);
+
+	/*
+	 * Other NSPU clients can use other buffers. Let's tell NSPU we use the
+	 * default buffer.
+	 */
+	buffaddr = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_DEFAULT_BUF));
+	NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_BUFFER)) = buffaddr;
+
+	/* NFP internal addresses are 40 bits. Clean all other bits here */
+	buffaddr = buffaddr & (((uint64_t)1 << 40) - 1);
+	desc->bufaddr = buffaddr;
+
+	/* Lets get information about the buffer */
+	nsp_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_DEFAULT_BUF_CFG));
+
+	/* Buffer size comes in MBs. Coversion to bytes */
+	desc->buf_size = ((size_t)nsp_reg & NSP_BUFFER_CFG_SIZE_MASK) << 20;
+
 	return 0;
 }
+
+#define NSPU_NFP_BUF(addr, base, off) \
+	(*(uint64_t *)((uint8_t *)(addr)->mem_base + ((base) | (off))))
+
+#define NSPU_HOST_BUF(base, off) (*(uint64_t *)((uint8_t *)(base) + (off)))
+
+static int
+nspu_buff_write(nspu_desc_t *desc, void *buffer, size_t size)
+{
+	uint64_t pcie_offset, pcie_window_base, pcie_window_offset;
+	uint64_t windowsz = desc->windowsz;
+	uint64_t buffaddr, j, i = 0;
+	int ret = 0;
+
+	if (size > desc->buf_size)
+		return -1;
+
+	buffaddr = desc->bufaddr;
+	windowsz = desc->windowsz;
+
+	while (i < size) {
+		/* Expansion bar reconfiguration per window size */
+		nspu_xlate(desc, buffaddr + i, &pcie_offset);
+		pcie_window_base = pcie_offset & (~(windowsz - 1));
+		pcie_window_offset = pcie_offset & (windowsz - 1);
+		for (j = pcie_window_offset; ((j < windowsz) && (i < size));
+		     j += 8) {
+			NSPU_NFP_BUF(desc, pcie_window_base, j) =
+				NSPU_HOST_BUF(buffer, i);
+			i += 8;
+		}
+	}
+
+	return ret;
+}
+
+static int
+nspu_buff_read(nspu_desc_t *desc, void *buffer, size_t size)
+{
+	uint64_t pcie_offset, pcie_window_base, pcie_window_offset;
+	uint64_t windowsz, i = 0, j;
+	uint64_t buffaddr;
+	int ret = 0;
+
+	if (size > desc->buf_size)
+		return -1;
+
+	buffaddr = desc->bufaddr;
+	windowsz = desc->windowsz;
+
+	while (i < size) {
+		/* Expansion bar reconfiguration per window size */
+		nspu_xlate(desc, buffaddr + i, &pcie_offset);
+		pcie_window_base = pcie_offset & (~(windowsz - 1));
+		pcie_window_offset = pcie_offset & (windowsz - 1);
+		for (j = pcie_window_offset; ((j < windowsz) && (i < size));
+		     j += 8) {
+			NSPU_HOST_BUF(buffer, i) =
+				NSPU_NFP_BUF(desc, pcie_window_base, j);
+			i += 8;
+		}
+	}
+
+	return ret;
+}
+
+static int
+nspu_command(nspu_desc_t *desc, uint16_t cmd, int read, int write,
+		 void *buffer, size_t rsize, size_t wsize)
+{
+	uint64_t status, cmd_reg;
+	uint64_t offset;
+	int retry = 0;
+	int retries = 120;
+	int ret = 0;
+
+	/* Same expansion BAR is used for different things */
+	nspu_xlate(desc, NSP_BASE, &offset);
+
+	status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS));
+
+	while ((status & 0x1) && (retry < retries)) {
+		status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS));
+		retry++;
+		sleep(1);
+	}
+
+	if (retry == retries)
+		return -1;
+
+	if (write) {
+		ret = nspu_buff_write(desc, buffer, wsize);
+		if (ret)
+			return ret;
+
+		/* Expansion BAR changes when writing the buffer */
+		nspu_xlate(desc, NSP_BASE, &offset);
+	}
+
+	NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND)) =
+		(uint64_t)wsize << 32 | (uint64_t)cmd << 16 | 1;
+
+	retry = 0;
+
+	cmd_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND));
+	while ((cmd_reg & 0x1) && (retry < retries)) {
+		cmd_reg = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_COMMAND));
+		retry++;
+		sleep(1);
+	}
+	if (retry == retries)
+		return -1;
+
+	retry = 0;
+	status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS));
+	while ((status & 0x1) && (retry < retries)) {
+		status = NSP_REG_VAL(NSP_REG_ADDR(desc, offset, NSP_STATUS));
+		retry++;
+		sleep(1);
+	}
+
+	if (retry == retries)
+		return -1;
+
+	ret = status & (0xff << 8);
+	if (ret)
+		return ret;
+
+	if (read) {
+		ret = nspu_buff_read(desc, buffer, rsize);
+		if (ret)
+			return ret;
+	}
+
+	return ret;
+}
+
+int
+nfp_fw_reset(nspu_desc_t *nspu_desc)
+{
+	int res;
+
+	res = nspu_command(nspu_desc, NSP_CMD_RESET, 0, 0, 0, 0, 0);
+
+	if (res < 0)
+		RTE_LOG(INFO, PMD, "fw reset failed: error %d", res);
+
+	return res;
+}
diff --git a/drivers/net/nfp/nfp_nspu.h b/drivers/net/nfp/nfp_nspu.h
index 7a1ac91..a142eb3 100644
--- a/drivers/net/nfp/nfp_nspu.h
+++ b/drivers/net/nfp/nfp_nspu.h
@@ -62,6 +62,9 @@
 	int pcie_bar;   /* PF PCI BAR to work with */
 	int exp_bar;    /* Expansion BAR number used by NSPU */
 	int barsz;      /* PCIE BAR log2 size */
+	uint64_t bufaddr;  /* commands buffer address */
+	size_t buf_size;   /* commands buffer size */
+	uint64_t windowsz; /* NSPU BAR window size */
 	void *cfg_base; /* Expansion BARs address */
 	void *mem_base; /* NSP interface */
 } nspu_desc_t;
@@ -69,3 +72,4 @@
 int nfp_nspu_init(nspu_desc_t *desc, int nfp, int pcie_bar, size_t pcie_barsz,
 		  int exp_bar, void *exp_bar_cfg_base, void *exp_bar_mmap);
 int nfp_nsp_get_abi_version(nspu_desc_t *desc, int *major, int *minor);
+int nfp_fw_reset(nspu_desc_t *nspu_desc);
-- 
1.9.1