DPDK patches and discussions
 help / color / mirror / Atom feed
From: Serhii Iliushyk <sil-plv@napatech.com>
To: dev@dpdk.org
Cc: mko-plv@napatech.com, sil-plv@napatech.com, ckm@napatech.com,
	andrew.rybchenko@oktetlabs.ru, ferruh.yigit@amd.com,
	stephen@networkplumber.org,
	Danylo Vodopianov <dvo-plv@napatech.com>
Subject: [PATCH v2 54/73] net/ntnic: add statistics API
Date: Tue, 22 Oct 2024 18:55:11 +0200	[thread overview]
Message-ID: <20241022165541.3186140-55-sil-plv@napatech.com> (raw)
In-Reply-To: <20241022165541.3186140-1-sil-plv@napatech.com>

From: Danylo Vodopianov <dvo-plv@napatech.com>

Statistics init, setup, get, reset APIs and their
implementation were added.

Statistics fpga defines were added.

Signed-off-by: Danylo Vodopianov <dvo-plv@napatech.com>
---
 drivers/net/ntnic/adapter/nt4ga_adapter.c     |  29 +-
 .../net/ntnic/adapter/nt4ga_stat/nt4ga_stat.c | 192 +++++++++
 .../net/ntnic/include/common_adapter_defs.h   |  15 +
 drivers/net/ntnic/include/create_elements.h   |   4 +
 drivers/net/ntnic/include/nt4ga_adapter.h     |   2 +
 drivers/net/ntnic/include/ntdrv_4ga.h         |   1 +
 drivers/net/ntnic/include/ntnic_stat.h        | 149 +++++++
 drivers/net/ntnic/include/ntos_drv.h          |   9 +
 .../ntnic/include/stream_binary_flow_api.h    |   5 +
 drivers/net/ntnic/meson.build                 |   3 +
 .../net/ntnic/nthw/core/include/nthw_rmc.h    |   1 +
 drivers/net/ntnic/nthw/core/nthw_rmc.c        |  10 +
 drivers/net/ntnic/nthw/stat/nthw_stat.c       | 370 ++++++++++++++++++
 .../ntnic/nthw/supported/nthw_fpga_mod_defs.h |   1 +
 .../ntnic/nthw/supported/nthw_fpga_reg_defs.h |   1 +
 .../nthw/supported/nthw_fpga_reg_defs_sta.h   |  40 ++
 drivers/net/ntnic/ntnic_ethdev.c              | 119 +++++-
 drivers/net/ntnic/ntnic_filter/ntnic_filter.c | 132 +++++++
 drivers/net/ntnic/ntnic_mod_reg.c             |  30 ++
 drivers/net/ntnic/ntnic_mod_reg.h             |  17 +
 drivers/net/ntnic/ntutil/nt_util.h            |   1 +
 21 files changed, 1119 insertions(+), 12 deletions(-)
 create mode 100644 drivers/net/ntnic/adapter/nt4ga_stat/nt4ga_stat.c
 create mode 100644 drivers/net/ntnic/include/common_adapter_defs.h
 create mode 100644 drivers/net/ntnic/nthw/stat/nthw_stat.c
 create mode 100644 drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs_sta.h

diff --git a/drivers/net/ntnic/adapter/nt4ga_adapter.c b/drivers/net/ntnic/adapter/nt4ga_adapter.c
index d9e6716c30..fa72dfda8d 100644
--- a/drivers/net/ntnic/adapter/nt4ga_adapter.c
+++ b/drivers/net/ntnic/adapter/nt4ga_adapter.c
@@ -212,19 +212,26 @@ static int nt4ga_adapter_init(struct adapter_info_s *p_adapter_info)
 		}
 	}
 
-	nthw_rmc_t *p_nthw_rmc = nthw_rmc_new();
-	if (p_nthw_rmc == NULL) {
-		NT_LOG(ERR, NTNIC, "Failed to allocate memory for RMC module");
-		return -1;
-	}
+	const struct nt4ga_stat_ops *nt4ga_stat_ops = get_nt4ga_stat_ops();
 
-	res = nthw_rmc_init(p_nthw_rmc, p_fpga, 0);
-	if (res) {
-		NT_LOG(ERR, NTNIC, "Failed to initialize RMC module");
-		return -1;
-	}
+	if (nt4ga_stat_ops != NULL) {
+		/* Nt4ga Stat init/setup */
+		res = nt4ga_stat_ops->nt4ga_stat_init(p_adapter_info);
+
+		if (res != 0) {
+			NT_LOG(ERR, NTNIC, "%s: Cannot initialize the statistics module",
+				p_adapter_id_str);
+			return res;
+		}
+
+		res = nt4ga_stat_ops->nt4ga_stat_setup(p_adapter_info);
 
-	nthw_rmc_unblock(p_nthw_rmc, false);
+		if (res != 0) {
+			NT_LOG(ERR, NTNIC, "%s: Cannot setup the statistics module",
+				p_adapter_id_str);
+			return res;
+		}
+	}
 
 	return 0;
 }
diff --git a/drivers/net/ntnic/adapter/nt4ga_stat/nt4ga_stat.c b/drivers/net/ntnic/adapter/nt4ga_stat/nt4ga_stat.c
new file mode 100644
index 0000000000..0e20f3ea45
--- /dev/null
+++ b/drivers/net/ntnic/adapter/nt4ga_stat/nt4ga_stat.c
@@ -0,0 +1,192 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include "ntlog.h"
+#include "nt_util.h"
+#include "nthw_drv.h"
+#include "nthw_fpga.h"
+#include "nthw_fpga_param_defs.h"
+#include "nt4ga_adapter.h"
+#include "ntnic_nim.h"
+#include "flow_filter.h"
+#include "ntnic_mod_reg.h"
+
+#define DEFAULT_MAX_BPS_SPEED 100e9
+
+static int nt4ga_stat_init(struct adapter_info_s *p_adapter_info)
+{
+	const char *const p_adapter_id_str = p_adapter_info->mp_adapter_id_str;
+	fpga_info_t *fpga_info = &p_adapter_info->fpga_info;
+	nthw_fpga_t *p_fpga = fpga_info->mp_fpga;
+	nt4ga_stat_t *p_nt4ga_stat = &p_adapter_info->nt4ga_stat;
+
+	if (p_nt4ga_stat) {
+		memset(p_nt4ga_stat, 0, sizeof(nt4ga_stat_t));
+
+	} else {
+		NT_LOG_DBGX(ERR, NTNIC, "%s: ERROR", p_adapter_id_str);
+		return -1;
+	}
+
+	{
+		nthw_stat_t *p_nthw_stat = nthw_stat_new();
+
+		if (!p_nthw_stat) {
+			NT_LOG_DBGX(ERR, NTNIC, "%s: ERROR", p_adapter_id_str);
+			return -1;
+		}
+
+		if (nthw_rmc_init(NULL, p_fpga, 0) == 0) {
+			nthw_rmc_t *p_nthw_rmc = nthw_rmc_new();
+
+			if (!p_nthw_rmc) {
+				nthw_stat_delete(p_nthw_stat);
+				NT_LOG(ERR, NTNIC, "%s: ERROR ", p_adapter_id_str);
+				return -1;
+			}
+
+			nthw_rmc_init(p_nthw_rmc, p_fpga, 0);
+			p_nt4ga_stat->mp_nthw_rmc = p_nthw_rmc;
+
+		} else {
+			p_nt4ga_stat->mp_nthw_rmc = NULL;
+		}
+
+		p_nt4ga_stat->mp_nthw_stat = p_nthw_stat;
+		nthw_stat_init(p_nthw_stat, p_fpga, 0);
+
+		p_nt4ga_stat->mn_rx_host_buffers = p_nthw_stat->m_nb_rx_host_buffers;
+		p_nt4ga_stat->mn_tx_host_buffers = p_nthw_stat->m_nb_tx_host_buffers;
+
+		p_nt4ga_stat->mn_rx_ports = p_nthw_stat->m_nb_rx_ports;
+		p_nt4ga_stat->mn_tx_ports = p_nthw_stat->m_nb_tx_ports;
+	}
+
+	return 0;
+}
+
+static int nt4ga_stat_setup(struct adapter_info_s *p_adapter_info)
+{
+	const int n_physical_adapter_no = p_adapter_info->adapter_no;
+	(void)n_physical_adapter_no;
+	nt4ga_stat_t *p_nt4ga_stat = &p_adapter_info->nt4ga_stat;
+	nthw_stat_t *p_nthw_stat = p_nt4ga_stat->mp_nthw_stat;
+
+	if (p_nt4ga_stat->mp_nthw_rmc)
+		nthw_rmc_block(p_nt4ga_stat->mp_nthw_rmc);
+
+	/* Allocate and map memory for fpga statistics */
+	{
+		uint32_t n_stat_size = (uint32_t)(p_nthw_stat->m_nb_counters * sizeof(uint32_t) +
+				sizeof(p_nthw_stat->mp_timestamp));
+		struct nt_dma_s *p_dma;
+		int numa_node = p_adapter_info->fpga_info.numa_node;
+
+		/* FPGA needs a 16K alignment on Statistics */
+		p_dma = nt_dma_alloc(n_stat_size, 0x4000, numa_node);
+
+		if (!p_dma) {
+			NT_LOG_DBGX(ERR, NTNIC, "p_dma alloc failed");
+			return -1;
+		}
+
+		NT_LOG_DBGX(DBG, NTNIC, "%x @%d %" PRIx64 " %" PRIx64, n_stat_size, numa_node,
+			p_dma->addr, p_dma->iova);
+
+		NT_LOG(DBG, NTNIC,
+			"DMA: Physical adapter %02d, PA = 0x%016" PRIX64 " DMA = 0x%016" PRIX64
+			" size = 0x%" PRIX32 "",
+			n_physical_adapter_no, p_dma->iova, p_dma->addr, n_stat_size);
+
+		p_nt4ga_stat->p_stat_dma_virtual = (uint32_t *)p_dma->addr;
+		p_nt4ga_stat->n_stat_size = n_stat_size;
+		p_nt4ga_stat->p_stat_dma = p_dma;
+
+		memset(p_nt4ga_stat->p_stat_dma_virtual, 0xaa, n_stat_size);
+		nthw_stat_set_dma_address(p_nthw_stat, p_dma->iova,
+			p_nt4ga_stat->p_stat_dma_virtual);
+	}
+
+	if (p_nt4ga_stat->mp_nthw_rmc)
+		nthw_rmc_unblock(p_nt4ga_stat->mp_nthw_rmc, false);
+
+	p_nt4ga_stat->mp_stat_structs_color =
+		calloc(p_nthw_stat->m_nb_color_counters, sizeof(struct color_counters));
+
+	if (!p_nt4ga_stat->mp_stat_structs_color) {
+		NT_LOG_DBGX(ERR, GENERAL, "Cannot allocate mem.");
+		return -1;
+	}
+
+	p_nt4ga_stat->mp_stat_structs_hb =
+		calloc(p_nt4ga_stat->mn_rx_host_buffers + p_nt4ga_stat->mn_tx_host_buffers,
+			sizeof(struct host_buffer_counters));
+
+	if (!p_nt4ga_stat->mp_stat_structs_hb) {
+		NT_LOG_DBGX(ERR, GENERAL, "Cannot allocate mem.");
+		return -1;
+	}
+
+	p_nt4ga_stat->cap.mp_stat_structs_port_rx =
+		calloc(NUM_ADAPTER_PORTS_MAX, sizeof(struct port_counters_v2));
+
+	if (!p_nt4ga_stat->cap.mp_stat_structs_port_rx) {
+		NT_LOG_DBGX(ERR, GENERAL, "Cannot allocate mem.");
+		return -1;
+	}
+
+	p_nt4ga_stat->cap.mp_stat_structs_port_tx =
+		calloc(NUM_ADAPTER_PORTS_MAX, sizeof(struct port_counters_v2));
+
+	if (!p_nt4ga_stat->cap.mp_stat_structs_port_tx) {
+		NT_LOG_DBGX(ERR, GENERAL, "Cannot allocate mem.");
+		return -1;
+	}
+
+	p_nt4ga_stat->mp_port_load =
+		calloc(NUM_ADAPTER_PORTS_MAX, sizeof(struct port_load_counters));
+
+	if (!p_nt4ga_stat->mp_port_load) {
+		NT_LOG_DBGX(ERR, GENERAL, "Cannot allocate mem.");
+		return -1;
+	}
+
+#ifdef NIM_TRIGGER
+	uint64_t max_bps_speed = nt_get_max_link_speed(p_adapter_info->nt4ga_link.speed_capa);
+
+	if (max_bps_speed == 0)
+		max_bps_speed = DEFAULT_MAX_BPS_SPEED;
+
+#else
+	uint64_t max_bps_speed = DEFAULT_MAX_BPS_SPEED;
+	NT_LOG(ERR, NTNIC, "NIM module not included");
+#endif
+
+	for (int p = 0; p < NUM_ADAPTER_PORTS_MAX; p++) {
+		p_nt4ga_stat->mp_port_load[p].rx_bps_max = max_bps_speed;
+		p_nt4ga_stat->mp_port_load[p].tx_bps_max = max_bps_speed;
+		p_nt4ga_stat->mp_port_load[p].rx_pps_max = max_bps_speed / (8 * (20 + 64));
+		p_nt4ga_stat->mp_port_load[p].tx_pps_max = max_bps_speed / (8 * (20 + 64));
+	}
+
+	memset(p_nt4ga_stat->a_stat_structs_color_base, 0,
+		sizeof(struct color_counters) * NT_MAX_COLOR_FLOW_STATS);
+	p_nt4ga_stat->last_timestamp = 0;
+
+	nthw_stat_trigger(p_nthw_stat);
+
+	return 0;
+}
+
+static struct nt4ga_stat_ops ops = {
+	.nt4ga_stat_init = nt4ga_stat_init,
+	.nt4ga_stat_setup = nt4ga_stat_setup,
+};
+
+void nt4ga_stat_ops_init(void)
+{
+	NT_LOG_DBGX(DBG, NTNIC, "Stat module was initialized");
+	register_nt4ga_stat_ops(&ops);
+}
diff --git a/drivers/net/ntnic/include/common_adapter_defs.h b/drivers/net/ntnic/include/common_adapter_defs.h
new file mode 100644
index 0000000000..6ed9121f0f
--- /dev/null
+++ b/drivers/net/ntnic/include/common_adapter_defs.h
@@ -0,0 +1,15 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef _COMMON_ADAPTER_DEFS_H_
+#define _COMMON_ADAPTER_DEFS_H_
+
+/*
+ * Declarations shared by NT adapter types.
+ */
+#define NUM_ADAPTER_MAX (8)
+#define NUM_ADAPTER_PORTS_MAX (128)
+
+#endif	/* _COMMON_ADAPTER_DEFS_H_ */
diff --git a/drivers/net/ntnic/include/create_elements.h b/drivers/net/ntnic/include/create_elements.h
index eaa578e72a..1456977837 100644
--- a/drivers/net/ntnic/include/create_elements.h
+++ b/drivers/net/ntnic/include/create_elements.h
@@ -46,6 +46,10 @@ struct rte_flow {
 
 	uint32_t flow_stat_id;
 
+	uint64_t stat_pkts;
+	uint64_t stat_bytes;
+	uint8_t stat_tcp_flags;
+
 	uint16_t caller_id;
 };
 
diff --git a/drivers/net/ntnic/include/nt4ga_adapter.h b/drivers/net/ntnic/include/nt4ga_adapter.h
index 809135f130..fef79ce358 100644
--- a/drivers/net/ntnic/include/nt4ga_adapter.h
+++ b/drivers/net/ntnic/include/nt4ga_adapter.h
@@ -6,6 +6,7 @@
 #ifndef _NT4GA_ADAPTER_H_
 #define _NT4GA_ADAPTER_H_
 
+#include "ntnic_stat.h"
 #include "nt4ga_link.h"
 
 typedef struct hw_info_s {
@@ -30,6 +31,7 @@ typedef struct hw_info_s {
 #include "ntnic_stat.h"
 
 typedef struct adapter_info_s {
+	struct nt4ga_stat_s nt4ga_stat;
 	struct nt4ga_filter_s nt4ga_filter;
 	struct nt4ga_link_s nt4ga_link;
 
diff --git a/drivers/net/ntnic/include/ntdrv_4ga.h b/drivers/net/ntnic/include/ntdrv_4ga.h
index 8ebdd98db0..1135e9a539 100644
--- a/drivers/net/ntnic/include/ntdrv_4ga.h
+++ b/drivers/net/ntnic/include/ntdrv_4ga.h
@@ -15,6 +15,7 @@ typedef struct ntdrv_4ga_s {
 
 	volatile bool b_shutdown;
 	rte_thread_t flm_thread;
+	pthread_mutex_t stat_lck;
 } ntdrv_4ga_t;
 
 #endif	/* __NTDRV_4GA_H__ */
diff --git a/drivers/net/ntnic/include/ntnic_stat.h b/drivers/net/ntnic/include/ntnic_stat.h
index 148088fe1d..2aee3f8425 100644
--- a/drivers/net/ntnic/include/ntnic_stat.h
+++ b/drivers/net/ntnic/include/ntnic_stat.h
@@ -6,6 +6,155 @@
 #ifndef NTNIC_STAT_H_
 #define NTNIC_STAT_H_
 
+#include "common_adapter_defs.h"
 #include "nthw_rmc.h"
+#include "nthw_fpga_model.h"
+
+#define NT_MAX_COLOR_FLOW_STATS 0x400
+
+struct nthw_stat {
+	nthw_fpga_t *mp_fpga;
+	nthw_module_t *mp_mod_stat;
+	int mn_instance;
+
+	int mn_stat_layout_version;
+
+	bool mb_has_tx_stats;
+
+	int m_nb_phy_ports;
+	int m_nb_nim_ports;
+
+	int m_nb_rx_ports;
+	int m_nb_tx_ports;
+
+	int m_nb_rx_host_buffers;
+	int m_nb_tx_host_buffers;
+
+	int m_dbs_present;
+
+	int m_rx_port_replicate;
+
+	int m_nb_color_counters;
+
+	int m_nb_rx_hb_counters;
+	int m_nb_tx_hb_counters;
+
+	int m_nb_rx_port_counters;
+	int m_nb_tx_port_counters;
+
+	int m_nb_counters;
+
+	int m_nb_rpp_per_ps;
+
+	nthw_field_t *mp_fld_dma_ena;
+	nthw_field_t *mp_fld_cnt_clear;
+
+	nthw_field_t *mp_fld_tx_disable;
+
+	nthw_field_t *mp_fld_cnt_freeze;
+
+	nthw_field_t *mp_fld_stat_toggle_missed;
+
+	nthw_field_t *mp_fld_dma_lsb;
+	nthw_field_t *mp_fld_dma_msb;
+
+	nthw_field_t *mp_fld_load_bin;
+	nthw_field_t *mp_fld_load_bps_rx0;
+	nthw_field_t *mp_fld_load_bps_rx1;
+	nthw_field_t *mp_fld_load_bps_tx0;
+	nthw_field_t *mp_fld_load_bps_tx1;
+	nthw_field_t *mp_fld_load_pps_rx0;
+	nthw_field_t *mp_fld_load_pps_rx1;
+	nthw_field_t *mp_fld_load_pps_tx0;
+	nthw_field_t *mp_fld_load_pps_tx1;
+
+	uint64_t m_stat_dma_physical;
+	uint32_t *mp_stat_dma_virtual;
+
+	uint64_t *mp_timestamp;
+};
+
+typedef struct nthw_stat nthw_stat_t;
+typedef struct nthw_stat nthw_stat;
+
+struct color_counters {
+	uint64_t color_packets;
+	uint64_t color_bytes;
+	uint8_t tcp_flags;
+};
+
+struct host_buffer_counters {
+};
+
+struct port_load_counters {
+	uint64_t rx_pps_max;
+	uint64_t tx_pps_max;
+	uint64_t rx_bps_max;
+	uint64_t tx_bps_max;
+};
+
+struct port_counters_v2 {
+};
+
+struct flm_counters_v1 {
+};
+
+struct nt4ga_stat_s {
+	nthw_stat_t *mp_nthw_stat;
+	nthw_rmc_t *mp_nthw_rmc;
+	struct nt_dma_s *p_stat_dma;
+	uint32_t *p_stat_dma_virtual;
+	uint32_t n_stat_size;
+
+	uint64_t last_timestamp;
+
+	int mn_rx_host_buffers;
+	int mn_tx_host_buffers;
+
+	int mn_rx_ports;
+	int mn_tx_ports;
+
+	struct color_counters *mp_stat_structs_color;
+	/* For calculating increments between stats polls */
+	struct color_counters a_stat_structs_color_base[NT_MAX_COLOR_FLOW_STATS];
+
+	/* Port counters for inline */
+	struct {
+		struct port_counters_v2 *mp_stat_structs_port_rx;
+		struct port_counters_v2 *mp_stat_structs_port_tx;
+	} cap;
+
+	struct host_buffer_counters *mp_stat_structs_hb;
+	struct port_load_counters *mp_port_load;
+
+	/* Rx/Tx totals: */
+	uint64_t n_totals_reset_timestamp;	/* timestamp for last totals reset */
+
+	uint64_t a_port_rx_octets_total[NUM_ADAPTER_PORTS_MAX];
+	/* Base is for calculating increments between statistics reads */
+	uint64_t a_port_rx_octets_base[NUM_ADAPTER_PORTS_MAX];
+
+	uint64_t a_port_rx_packets_total[NUM_ADAPTER_PORTS_MAX];
+	uint64_t a_port_rx_packets_base[NUM_ADAPTER_PORTS_MAX];
+
+	uint64_t a_port_rx_drops_total[NUM_ADAPTER_PORTS_MAX];
+	uint64_t a_port_rx_drops_base[NUM_ADAPTER_PORTS_MAX];
+
+	uint64_t a_port_tx_octets_total[NUM_ADAPTER_PORTS_MAX];
+	uint64_t a_port_tx_octets_base[NUM_ADAPTER_PORTS_MAX];
+
+	uint64_t a_port_tx_packets_base[NUM_ADAPTER_PORTS_MAX];
+	uint64_t a_port_tx_packets_total[NUM_ADAPTER_PORTS_MAX];
+};
+
+typedef struct nt4ga_stat_s nt4ga_stat_t;
+
+nthw_stat_t *nthw_stat_new(void);
+int nthw_stat_init(nthw_stat_t *p, nthw_fpga_t *p_fpga, int n_instance);
+void nthw_stat_delete(nthw_stat_t *p);
+
+int nthw_stat_set_dma_address(nthw_stat_t *p, uint64_t stat_dma_physical,
+	uint32_t *p_stat_dma_virtual);
+int nthw_stat_trigger(nthw_stat_t *p);
 
 #endif  /* NTNIC_STAT_H_ */
diff --git a/drivers/net/ntnic/include/ntos_drv.h b/drivers/net/ntnic/include/ntos_drv.h
index 8fd577dfe3..7b3c8ff3d6 100644
--- a/drivers/net/ntnic/include/ntos_drv.h
+++ b/drivers/net/ntnic/include/ntos_drv.h
@@ -57,6 +57,9 @@ struct __rte_cache_aligned ntnic_rx_queue {
 	struct flow_queue_id_s queue;    /* queue info - user id and hw queue index */
 	struct rte_mempool *mb_pool; /* mbuf memory pool */
 	uint16_t buf_size; /* Size of data area in mbuf */
+	unsigned long rx_pkts;	/* Rx packet statistics */
+	unsigned long rx_bytes;	/* Rx bytes statistics */
+	unsigned long err_pkts;	/* Rx error packet statistics */
 	int  enabled;  /* Enabling/disabling of this queue */
 
 	struct hwq_s           hwq;
@@ -80,6 +83,9 @@ struct __rte_cache_aligned ntnic_tx_queue {
 	int rss_target_id;
 
 	uint32_t port;     /* Tx port for this queue */
+	unsigned long tx_pkts;	/* Tx packet statistics */
+	unsigned long tx_bytes;	/* Tx bytes statistics */
+	unsigned long err_pkts;	/* Tx error packet stat */
 	int  enabled;  /* Enabling/disabling of this queue */
 	enum fpga_info_profile profile;  /* Inline / Capture */
 };
@@ -95,6 +101,7 @@ struct pmd_internals {
 	/* Offset of the VF from the PF */
 	uint8_t vf_offset;
 	uint32_t port;
+	uint32_t port_id;
 	nt_meta_port_type_t type;
 	struct flow_queue_id_s vpq[MAX_QUEUES];
 	unsigned int           vpq_nb_vq;
@@ -107,6 +114,8 @@ struct pmd_internals {
 	struct rte_ether_addr eth_addrs[NUM_MAC_ADDRS_PER_PORT];
 	/* Multicast ethernet (MAC) addresses. */
 	struct rte_ether_addr mc_addrs[NUM_MULTICAST_ADDRS_PER_PORT];
+	uint64_t last_stat_rtc;
+	uint64_t rx_missed;
 	struct pmd_internals *next;
 };
 
diff --git a/drivers/net/ntnic/include/stream_binary_flow_api.h b/drivers/net/ntnic/include/stream_binary_flow_api.h
index e5fe686d99..4ce1561033 100644
--- a/drivers/net/ntnic/include/stream_binary_flow_api.h
+++ b/drivers/net/ntnic/include/stream_binary_flow_api.h
@@ -6,6 +6,7 @@
 #ifndef _STREAM_BINARY_FLOW_API_H_
 #define _STREAM_BINARY_FLOW_API_H_
 
+#include <rte_ether.h>
 #include "rte_flow.h"
 #include "rte_flow_driver.h"
 
@@ -44,6 +45,10 @@
 #define FLOW_MAX_QUEUES 128
 
 #define RAW_ENCAP_DECAP_ELEMS_MAX 16
+
+extern uint64_t rte_tsc_freq;
+extern rte_spinlock_t hwlock;
+
 /*
  * Flow eth dev profile determines how the FPGA module resources are
  * managed and what features are available
diff --git a/drivers/net/ntnic/meson.build b/drivers/net/ntnic/meson.build
index 92167d24e4..216341bb11 100644
--- a/drivers/net/ntnic/meson.build
+++ b/drivers/net/ntnic/meson.build
@@ -25,10 +25,12 @@ includes = [
 # all sources
 sources = files(
         'adapter/nt4ga_adapter.c',
+        'adapter/nt4ga_stat/nt4ga_stat.c',
         'dbsconfig/ntnic_dbsconfig.c',
         'link_mgmt/link_100g/nt4ga_link_100g.c',
         'link_mgmt/nt4ga_link.c',
         'nim/i2c_nim.c',
+        'ntnic_filter/ntnic_filter.c',
         'nthw/dbs/nthw_dbs.c',
         'nthw/supported/nthw_fpga_9563_055_049_0000.c',
         'nthw/supported/nthw_fpga_instances.c',
@@ -48,6 +50,7 @@ sources = files(
         'nthw/core/nthw_rmc.c',
         'nthw/core/nthw_sdc.c',
         'nthw/core/nthw_si5340.c',
+        'nthw/stat/nthw_stat.c',
         'nthw/flow_api/flow_api.c',
         'nthw/flow_api/flow_group.c',
         'nthw/flow_api/flow_id_table.c',
diff --git a/drivers/net/ntnic/nthw/core/include/nthw_rmc.h b/drivers/net/ntnic/nthw/core/include/nthw_rmc.h
index 2345820bdc..b239752674 100644
--- a/drivers/net/ntnic/nthw/core/include/nthw_rmc.h
+++ b/drivers/net/ntnic/nthw/core/include/nthw_rmc.h
@@ -44,6 +44,7 @@ typedef struct nthw_rmc nthw_rmc;
 nthw_rmc_t *nthw_rmc_new(void);
 int nthw_rmc_init(nthw_rmc_t *p, nthw_fpga_t *p_fpga, int n_instance);
 
+void nthw_rmc_block(nthw_rmc_t *p);
 void nthw_rmc_unblock(nthw_rmc_t *p, bool b_is_secondary);
 
 #endif	/* NTHW_RMC_H_ */
diff --git a/drivers/net/ntnic/nthw/core/nthw_rmc.c b/drivers/net/ntnic/nthw/core/nthw_rmc.c
index 4a01424c24..748519aeb4 100644
--- a/drivers/net/ntnic/nthw/core/nthw_rmc.c
+++ b/drivers/net/ntnic/nthw/core/nthw_rmc.c
@@ -77,6 +77,16 @@ int nthw_rmc_init(nthw_rmc_t *p, nthw_fpga_t *p_fpga, int n_instance)
 	return 0;
 }
 
+void nthw_rmc_block(nthw_rmc_t *p)
+{
+	/* BLOCK_STATT(0)=1 BLOCK_KEEPA(1)=1 BLOCK_MAC_PORT(8:11)=~0 */
+	if (!p->mb_administrative_block) {
+		nthw_field_set_flush(p->mp_fld_ctrl_block_stat_drop);
+		nthw_field_set_flush(p->mp_fld_ctrl_block_keep_alive);
+		nthw_field_set_flush(p->mp_fld_ctrl_block_mac_port);
+	}
+}
+
 void nthw_rmc_unblock(nthw_rmc_t *p, bool b_is_secondary)
 {
 	uint32_t n_block_mask = ~0U << (b_is_secondary ? p->mn_nims : p->mn_ports);
diff --git a/drivers/net/ntnic/nthw/stat/nthw_stat.c b/drivers/net/ntnic/nthw/stat/nthw_stat.c
new file mode 100644
index 0000000000..6adcd2e090
--- /dev/null
+++ b/drivers/net/ntnic/nthw/stat/nthw_stat.c
@@ -0,0 +1,370 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include "nt_util.h"
+#include "ntlog.h"
+
+#include "nthw_drv.h"
+#include "nthw_register.h"
+
+#include "ntnic_stat.h"
+
+#include <malloc.h>
+
+nthw_stat_t *nthw_stat_new(void)
+{
+	nthw_stat_t *p = malloc(sizeof(nthw_stat_t));
+
+	if (p)
+		memset(p, 0, sizeof(nthw_stat_t));
+
+	return p;
+}
+
+void nthw_stat_delete(nthw_stat_t *p)
+{
+	if (p)
+		free(p);
+}
+
+int nthw_stat_init(nthw_stat_t *p, nthw_fpga_t *p_fpga, int n_instance)
+{
+	const char *const p_adapter_id_str = p_fpga->p_fpga_info->mp_adapter_id_str;
+	uint64_t n_module_version_packed64 = -1;
+	nthw_module_t *mod = nthw_fpga_query_module(p_fpga, MOD_STA, n_instance);
+
+	if (p == NULL)
+		return mod == NULL ? -1 : 0;
+
+	if (mod == NULL) {
+		NT_LOG(ERR, NTHW, "%s: STAT %d: no such instance", p_adapter_id_str, n_instance);
+		return -1;
+	}
+
+	p->mp_fpga = p_fpga;
+	p->mn_instance = n_instance;
+	p->mp_mod_stat = mod;
+
+	n_module_version_packed64 = nthw_module_get_version_packed64(p->mp_mod_stat);
+	NT_LOG(DBG, NTHW, "%s: STAT %d: version=0x%08lX", p_adapter_id_str, p->mn_instance,
+		n_module_version_packed64);
+
+	{
+		nthw_register_t *p_reg;
+		/* STA_CFG register */
+		p_reg = nthw_module_get_register(p->mp_mod_stat, STA_CFG);
+		p->mp_fld_dma_ena = nthw_register_get_field(p_reg, STA_CFG_DMA_ENA);
+		p->mp_fld_cnt_clear = nthw_register_get_field(p_reg, STA_CFG_CNT_CLEAR);
+
+		/* CFG: fields NOT available from v. 3 */
+		p->mp_fld_tx_disable = nthw_register_query_field(p_reg, STA_CFG_TX_DISABLE);
+		p->mp_fld_cnt_freeze = nthw_register_query_field(p_reg, STA_CFG_CNT_FRZ);
+
+		/* STA_STATUS register */
+		p_reg = nthw_module_get_register(p->mp_mod_stat, STA_STATUS);
+		p->mp_fld_stat_toggle_missed =
+			nthw_register_get_field(p_reg, STA_STATUS_STAT_TOGGLE_MISSED);
+
+		/* HOST_ADR registers */
+		p_reg = nthw_module_get_register(p->mp_mod_stat, STA_HOST_ADR_LSB);
+		p->mp_fld_dma_lsb = nthw_register_get_field(p_reg, STA_HOST_ADR_LSB_LSB);
+
+		p_reg = nthw_module_get_register(p->mp_mod_stat, STA_HOST_ADR_MSB);
+		p->mp_fld_dma_msb = nthw_register_get_field(p_reg, STA_HOST_ADR_MSB_MSB);
+
+		/* Binning cycles */
+		p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_BIN);
+
+		if (p_reg) {
+			p->mp_fld_load_bin = nthw_register_get_field(p_reg, STA_LOAD_BIN_BIN);
+
+			/* Bandwidth load for RX port 0 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_BPS_RX_0);
+
+			if (p_reg) {
+				p->mp_fld_load_bps_rx0 =
+					nthw_register_get_field(p_reg, STA_LOAD_BPS_RX_0_BPS);
+
+			} else {
+				p->mp_fld_load_bps_rx0 = NULL;
+			}
+
+			/* Bandwidth load for RX port 1 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_BPS_RX_1);
+
+			if (p_reg) {
+				p->mp_fld_load_bps_rx1 =
+					nthw_register_get_field(p_reg, STA_LOAD_BPS_RX_1_BPS);
+
+			} else {
+				p->mp_fld_load_bps_rx1 = NULL;
+			}
+
+			/* Bandwidth load for TX port 0 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_BPS_TX_0);
+
+			if (p_reg) {
+				p->mp_fld_load_bps_tx0 =
+					nthw_register_get_field(p_reg, STA_LOAD_BPS_TX_0_BPS);
+
+			} else {
+				p->mp_fld_load_bps_tx0 = NULL;
+			}
+
+			/* Bandwidth load for TX port 1 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_BPS_TX_1);
+
+			if (p_reg) {
+				p->mp_fld_load_bps_tx1 =
+					nthw_register_get_field(p_reg, STA_LOAD_BPS_TX_1_BPS);
+
+			} else {
+				p->mp_fld_load_bps_tx1 = NULL;
+			}
+
+			/* Packet load for RX port 0 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_PPS_RX_0);
+
+			if (p_reg) {
+				p->mp_fld_load_pps_rx0 =
+					nthw_register_get_field(p_reg, STA_LOAD_PPS_RX_0_PPS);
+
+			} else {
+				p->mp_fld_load_pps_rx0 = NULL;
+			}
+
+			/* Packet load for RX port 1 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_PPS_RX_1);
+
+			if (p_reg) {
+				p->mp_fld_load_pps_rx1 =
+					nthw_register_get_field(p_reg, STA_LOAD_PPS_RX_1_PPS);
+
+			} else {
+				p->mp_fld_load_pps_rx1 = NULL;
+			}
+
+			/* Packet load for TX port 0 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_PPS_TX_0);
+
+			if (p_reg) {
+				p->mp_fld_load_pps_tx0 =
+					nthw_register_get_field(p_reg, STA_LOAD_PPS_TX_0_PPS);
+
+			} else {
+				p->mp_fld_load_pps_tx0 = NULL;
+			}
+
+			/* Packet load for TX port 1 */
+			p_reg = nthw_module_query_register(p->mp_mod_stat, STA_LOAD_PPS_TX_1);
+
+			if (p_reg) {
+				p->mp_fld_load_pps_tx1 =
+					nthw_register_get_field(p_reg, STA_LOAD_PPS_TX_1_PPS);
+
+			} else {
+				p->mp_fld_load_pps_tx1 = NULL;
+			}
+
+		} else {
+			p->mp_fld_load_bin = NULL;
+			p->mp_fld_load_bps_rx0 = NULL;
+			p->mp_fld_load_bps_rx1 = NULL;
+			p->mp_fld_load_bps_tx0 = NULL;
+			p->mp_fld_load_bps_tx1 = NULL;
+			p->mp_fld_load_pps_rx0 = NULL;
+			p->mp_fld_load_pps_rx1 = NULL;
+			p->mp_fld_load_pps_tx0 = NULL;
+			p->mp_fld_load_pps_tx1 = NULL;
+		}
+	}
+
+	/* Params */
+	p->m_nb_nim_ports = nthw_fpga_get_product_param(p_fpga, NT_NIMS, 0);
+	p->m_nb_phy_ports = nthw_fpga_get_product_param(p_fpga, NT_PHY_PORTS, 0);
+
+	/* VSWITCH */
+	p->m_nb_rx_ports = nthw_fpga_get_product_param(p_fpga, NT_STA_RX_PORTS, -1);
+
+	if (p->m_nb_rx_ports == -1) {
+		/* non-VSWITCH */
+		p->m_nb_rx_ports = nthw_fpga_get_product_param(p_fpga, NT_RX_PORTS, -1);
+
+		if (p->m_nb_rx_ports == -1) {
+			/* non-VSWITCH */
+			p->m_nb_rx_ports = nthw_fpga_get_product_param(p_fpga, NT_PORTS, 0);
+		}
+	}
+
+	p->m_nb_rpp_per_ps = nthw_fpga_get_product_param(p_fpga, NT_RPP_PER_PS, 0);
+
+	p->m_nb_tx_ports = nthw_fpga_get_product_param(p_fpga, NT_TX_PORTS, 0);
+	p->m_rx_port_replicate = nthw_fpga_get_product_param(p_fpga, NT_RX_PORT_REPLICATE, 0);
+
+	/* VSWITCH */
+	p->m_nb_color_counters = nthw_fpga_get_product_param(p_fpga, NT_STA_COLORS, 64) * 2;
+
+	if (p->m_nb_color_counters == 0) {
+		/* non-VSWITCH */
+		p->m_nb_color_counters = nthw_fpga_get_product_param(p_fpga, NT_CAT_FUNCS, 0) * 2;
+	}
+
+	p->m_nb_rx_host_buffers = nthw_fpga_get_product_param(p_fpga, NT_QUEUES, 0);
+	p->m_nb_tx_host_buffers = p->m_nb_rx_host_buffers;
+
+	p->m_dbs_present = nthw_fpga_get_product_param(p_fpga, NT_DBS_PRESENT, 0);
+
+	p->m_nb_rx_hb_counters = (p->m_nb_rx_host_buffers * (6 + 2 *
+				(n_module_version_packed64 >= VERSION_PACKED64(0, 6) ?
+					p->m_dbs_present : 0)));
+
+	p->m_nb_tx_hb_counters = 0;
+
+	p->m_nb_rx_port_counters = 42 +
+		2 * (n_module_version_packed64 >= VERSION_PACKED64(0, 6) ? p->m_dbs_present : 0);
+	p->m_nb_tx_port_counters = 0;
+
+	p->m_nb_counters =
+		p->m_nb_color_counters + p->m_nb_rx_hb_counters + p->m_nb_tx_hb_counters;
+
+	p->mn_stat_layout_version = 0;
+
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 9)) {
+		p->mn_stat_layout_version = 7;
+
+	} else if (n_module_version_packed64 >= VERSION_PACKED64(0, 8)) {
+		p->mn_stat_layout_version = 6;
+
+	} else if (n_module_version_packed64 >= VERSION_PACKED64(0, 6)) {
+		p->mn_stat_layout_version = 5;
+
+	} else if (n_module_version_packed64 >= VERSION_PACKED64(0, 4)) {
+		p->mn_stat_layout_version = 4;
+
+	} else if (n_module_version_packed64 >= VERSION_PACKED64(0, 3)) {
+		p->mn_stat_layout_version = 3;
+
+	} else if (n_module_version_packed64 >= VERSION_PACKED64(0, 2)) {
+		p->mn_stat_layout_version = 2;
+
+	} else if (n_module_version_packed64 > VERSION_PACKED64(0, 0)) {
+		p->mn_stat_layout_version = 1;
+
+	} else {
+		p->mn_stat_layout_version = 0;
+		NT_LOG(ERR, NTHW, "%s: unknown module_version 0x%08lX layout=%d",
+			p_adapter_id_str, n_module_version_packed64, p->mn_stat_layout_version);
+	}
+
+	assert(p->mn_stat_layout_version);
+
+	/* STA module 0.2+ adds IPF counters per port (Rx feature) */
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 2))
+		p->m_nb_rx_port_counters += 6;
+
+	/* STA module 0.3+ adds TX stats */
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 3) || p->m_nb_tx_ports >= 1)
+		p->mb_has_tx_stats = true;
+
+	/* STA module 0.3+ adds TX stat counters */
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 3))
+		p->m_nb_tx_port_counters += 22;
+
+	/* STA module 0.4+ adds TX drop event counter */
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 4))
+		p->m_nb_tx_port_counters += 1;	/* TX drop event counter */
+
+	/*
+	 * STA module 0.6+ adds pkt filter drop octets+pkts, retransmit and
+	 * duplicate counters
+	 */
+	if (n_module_version_packed64 >= VERSION_PACKED64(0, 6)) {
+		p->m_nb_rx_port_counters += 4;
+		p->m_nb_tx_port_counters += 1;
+	}
+
+	p->m_nb_counters += (p->m_nb_rx_ports * p->m_nb_rx_port_counters);
+
+	if (p->mb_has_tx_stats)
+		p->m_nb_counters += (p->m_nb_tx_ports * p->m_nb_tx_port_counters);
+
+	/* Output params (debug) */
+	NT_LOG(DBG, NTHW, "%s: nims=%d rxports=%d txports=%d rxrepl=%d colors=%d queues=%d",
+		p_adapter_id_str, p->m_nb_nim_ports, p->m_nb_rx_ports, p->m_nb_tx_ports,
+		p->m_rx_port_replicate, p->m_nb_color_counters, p->m_nb_rx_host_buffers);
+	NT_LOG(DBG, NTHW, "%s: hbs=%d hbcounters=%d rxcounters=%d txcounters=%d",
+		p_adapter_id_str, p->m_nb_rx_host_buffers, p->m_nb_rx_hb_counters,
+		p->m_nb_rx_port_counters, p->m_nb_tx_port_counters);
+	NT_LOG(DBG, NTHW, "%s: layout=%d", p_adapter_id_str, p->mn_stat_layout_version);
+	NT_LOG(DBG, NTHW, "%s: counters=%d (0x%X)", p_adapter_id_str, p->m_nb_counters,
+		p->m_nb_counters);
+
+	/* Init */
+	if (p->mp_fld_tx_disable)
+		nthw_field_set_flush(p->mp_fld_tx_disable);
+
+	nthw_field_update_register(p->mp_fld_cnt_clear);
+	nthw_field_set_flush(p->mp_fld_cnt_clear);
+	nthw_field_clr_flush(p->mp_fld_cnt_clear);
+
+	nthw_field_update_register(p->mp_fld_stat_toggle_missed);
+	nthw_field_set_flush(p->mp_fld_stat_toggle_missed);
+
+	nthw_field_update_register(p->mp_fld_dma_ena);
+	nthw_field_clr_flush(p->mp_fld_dma_ena);
+	nthw_field_update_register(p->mp_fld_dma_ena);
+
+	/* Set the sliding windows size for port load */
+	if (p->mp_fld_load_bin) {
+		uint32_t rpp = nthw_fpga_get_product_param(p_fpga, NT_RPP_PER_PS, 0);
+		uint32_t bin =
+			(uint32_t)(((PORT_LOAD_WINDOWS_SIZE * 1000000000000ULL) / (32ULL * rpp)) -
+				1ULL);
+		nthw_field_set_val_flush32(p->mp_fld_load_bin, bin);
+	}
+
+	return 0;
+}
+
+int nthw_stat_set_dma_address(nthw_stat_t *p, uint64_t stat_dma_physical,
+	uint32_t *p_stat_dma_virtual)
+{
+	assert(p_stat_dma_virtual);
+	p->mp_timestamp = NULL;
+
+	p->m_stat_dma_physical = stat_dma_physical;
+	p->mp_stat_dma_virtual = p_stat_dma_virtual;
+
+	memset(p->mp_stat_dma_virtual, 0, (p->m_nb_counters * sizeof(uint32_t)));
+
+	nthw_field_set_val_flush32(p->mp_fld_dma_msb,
+		(uint32_t)((p->m_stat_dma_physical >> 32) & 0xffffffff));
+	nthw_field_set_val_flush32(p->mp_fld_dma_lsb,
+		(uint32_t)(p->m_stat_dma_physical & 0xffffffff));
+
+	p->mp_timestamp = (uint64_t *)(p->mp_stat_dma_virtual + p->m_nb_counters);
+	NT_LOG(DBG, NTHW,
+		"stat_dma_physical=%" PRIX64 " p_stat_dma_virtual=%" PRIX64
+		" mp_timestamp=%" PRIX64 "", p->m_stat_dma_physical,
+		(uint64_t)p->mp_stat_dma_virtual, (uint64_t)p->mp_timestamp);
+	*p->mp_timestamp = (uint64_t)(int64_t)-1;
+	return 0;
+}
+
+int nthw_stat_trigger(nthw_stat_t *p)
+{
+	int n_toggle_miss = nthw_field_get_updated(p->mp_fld_stat_toggle_missed);
+
+	if (n_toggle_miss)
+		nthw_field_set_flush(p->mp_fld_stat_toggle_missed);
+
+	if (p->mp_timestamp)
+		*p->mp_timestamp = -1;	/* Clear old ts */
+
+	nthw_field_update_register(p->mp_fld_dma_ena);
+	nthw_field_set_flush(p->mp_fld_dma_ena);
+
+	return 0;
+}
diff --git a/drivers/net/ntnic/nthw/supported/nthw_fpga_mod_defs.h b/drivers/net/ntnic/nthw/supported/nthw_fpga_mod_defs.h
index 2b059d98ff..ddc144dc02 100644
--- a/drivers/net/ntnic/nthw/supported/nthw_fpga_mod_defs.h
+++ b/drivers/net/ntnic/nthw/supported/nthw_fpga_mod_defs.h
@@ -46,6 +46,7 @@
 #define MOD_SDC (0xd2369530UL)
 #define MOD_SLC (0x1aef1f38UL)
 #define MOD_SLC_LR (0x969fc50bUL)
+#define MOD_STA (0x76fae64dUL)
 #define MOD_TX_CPY (0x60acf217UL)
 #define MOD_TX_INS (0x59afa100UL)
 #define MOD_TX_RPL (0x1095dfbbUL)
diff --git a/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs.h b/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs.h
index 7741aa563f..8f196f885f 100644
--- a/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs.h
+++ b/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs.h
@@ -45,6 +45,7 @@
 #include "nthw_fpga_reg_defs_sdc.h"
 #include "nthw_fpga_reg_defs_slc.h"
 #include "nthw_fpga_reg_defs_slc_lr.h"
+#include "nthw_fpga_reg_defs_sta.h"
 #include "nthw_fpga_reg_defs_tx_cpy.h"
 #include "nthw_fpga_reg_defs_tx_ins.h"
 #include "nthw_fpga_reg_defs_tx_rpl.h"
diff --git a/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs_sta.h b/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs_sta.h
new file mode 100644
index 0000000000..640ffcbc52
--- /dev/null
+++ b/drivers/net/ntnic/nthw/supported/nthw_fpga_reg_defs_sta.h
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2024 Napatech A/S
+ */
+
+#ifndef _NTHW_FPGA_REG_DEFS_STA_
+#define _NTHW_FPGA_REG_DEFS_STA_
+
+/* STA */
+#define STA_CFG (0xcecaf9f4UL)
+#define STA_CFG_CNT_CLEAR (0xc325e12eUL)
+#define STA_CFG_CNT_FRZ (0x8c27a596UL)
+#define STA_CFG_DMA_ENA (0x940dbacUL)
+#define STA_CFG_TX_DISABLE (0x30f43250UL)
+#define STA_HOST_ADR_LSB (0xde569336UL)
+#define STA_HOST_ADR_LSB_LSB (0xb6f2f94bUL)
+#define STA_HOST_ADR_MSB (0xdf94f901UL)
+#define STA_HOST_ADR_MSB_MSB (0x114798c8UL)
+#define STA_LOAD_BIN (0x2e842591UL)
+#define STA_LOAD_BIN_BIN (0x1a2b942eUL)
+#define STA_LOAD_BPS_RX_0 (0xbf8f4595UL)
+#define STA_LOAD_BPS_RX_0_BPS (0x41647781UL)
+#define STA_LOAD_BPS_RX_1 (0xc8887503UL)
+#define STA_LOAD_BPS_RX_1_BPS (0x7c045e31UL)
+#define STA_LOAD_BPS_TX_0 (0x9ae41a49UL)
+#define STA_LOAD_BPS_TX_0_BPS (0x870b7e06UL)
+#define STA_LOAD_BPS_TX_1 (0xede32adfUL)
+#define STA_LOAD_BPS_TX_1_BPS (0xba6b57b6UL)
+#define STA_LOAD_PPS_RX_0 (0x811173c3UL)
+#define STA_LOAD_PPS_RX_0_PPS (0xbee573fcUL)
+#define STA_LOAD_PPS_RX_1 (0xf6164355UL)
+#define STA_LOAD_PPS_RX_1_PPS (0x83855a4cUL)
+#define STA_LOAD_PPS_TX_0 (0xa47a2c1fUL)
+#define STA_LOAD_PPS_TX_0_PPS (0x788a7a7bUL)
+#define STA_LOAD_PPS_TX_1 (0xd37d1c89UL)
+#define STA_LOAD_PPS_TX_1_PPS (0x45ea53cbUL)
+#define STA_STATUS (0x91c5c51cUL)
+#define STA_STATUS_STAT_TOGGLE_MISSED (0xf7242b11UL)
+
+#endif	/* _NTHW_FPGA_REG_DEFS_STA_ */
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index 1b25621537..86876ecda6 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -65,6 +65,8 @@ const rte_thread_attr_t thread_attr = { .priority = RTE_THREAD_PRIORITY_NORMAL }
 #define MAX_RX_PACKETS   128
 #define MAX_TX_PACKETS   128
 
+uint64_t rte_tsc_freq;
+
 int kill_pmd;
 
 #define ETH_DEV_NTNIC_HELP_ARG "help"
@@ -88,7 +90,7 @@ static const struct rte_pci_id nthw_pci_id_map[] = {
 
 static const struct sg_ops_s *sg_ops;
 
-static rte_spinlock_t hwlock = RTE_SPINLOCK_INITIALIZER;
+rte_spinlock_t hwlock = RTE_SPINLOCK_INITIALIZER;
 
 /*
  * Store and get adapter info
@@ -156,6 +158,102 @@ get_pdrv_from_pci(struct rte_pci_addr addr)
 	return p_drv;
 }
 
+static int dpdk_stats_collect(struct pmd_internals *internals, struct rte_eth_stats *stats)
+{
+	const struct ntnic_filter_ops *ntnic_filter_ops = get_ntnic_filter_ops();
+
+	if (ntnic_filter_ops == NULL) {
+		NT_LOG_DBGX(ERR, NTNIC, "ntnic_filter_ops uninitialized");
+		return -1;
+	}
+
+	unsigned int i;
+	struct drv_s *p_drv = internals->p_drv;
+	struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+	nt4ga_stat_t *p_nt4ga_stat = &p_nt_drv->adapter_info.nt4ga_stat;
+	nthw_stat_t *p_nthw_stat = p_nt4ga_stat->mp_nthw_stat;
+	const int if_index = internals->n_intf_no;
+	uint64_t rx_total = 0;
+	uint64_t rx_total_b = 0;
+	uint64_t tx_total = 0;
+	uint64_t tx_total_b = 0;
+	uint64_t tx_err_total = 0;
+
+	if (!p_nthw_stat || !p_nt4ga_stat || !stats || if_index < 0 ||
+		if_index > NUM_ADAPTER_PORTS_MAX) {
+		NT_LOG_DBGX(WRN, NTNIC, "error exit");
+		return -1;
+	}
+
+	/*
+	 * Pull the latest port statistic numbers (Rx/Tx pkts and bytes)
+	 * Return values are in the "internals->rxq_scg[]" and "internals->txq_scg[]" arrays
+	 */
+	ntnic_filter_ops->poll_statistics(internals);
+
+	memset(stats, 0, sizeof(*stats));
+
+	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < internals->nb_rx_queues; i++) {
+		stats->q_ipackets[i] = internals->rxq_scg[i].rx_pkts;
+		stats->q_ibytes[i] = internals->rxq_scg[i].rx_bytes;
+		rx_total += stats->q_ipackets[i];
+		rx_total_b += stats->q_ibytes[i];
+	}
+
+	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < internals->nb_tx_queues; i++) {
+		stats->q_opackets[i] = internals->txq_scg[i].tx_pkts;
+		stats->q_obytes[i] = internals->txq_scg[i].tx_bytes;
+		stats->q_errors[i] = internals->txq_scg[i].err_pkts;
+		tx_total += stats->q_opackets[i];
+		tx_total_b += stats->q_obytes[i];
+		tx_err_total += stats->q_errors[i];
+	}
+
+	stats->imissed = internals->rx_missed;
+	stats->ipackets = rx_total;
+	stats->ibytes = rx_total_b;
+	stats->opackets = tx_total;
+	stats->obytes = tx_total_b;
+	stats->oerrors = tx_err_total;
+
+	return 0;
+}
+
+static int dpdk_stats_reset(struct pmd_internals *internals, struct ntdrv_4ga_s *p_nt_drv,
+	int n_intf_no)
+{
+	nt4ga_stat_t *p_nt4ga_stat = &p_nt_drv->adapter_info.nt4ga_stat;
+	nthw_stat_t *p_nthw_stat = p_nt4ga_stat->mp_nthw_stat;
+	unsigned int i;
+
+	if (!p_nthw_stat || !p_nt4ga_stat || n_intf_no < 0 || n_intf_no > NUM_ADAPTER_PORTS_MAX)
+		return -1;
+
+	pthread_mutex_lock(&p_nt_drv->stat_lck);
+
+	/* Rx */
+	for (i = 0; i < internals->nb_rx_queues; i++) {
+		internals->rxq_scg[i].rx_pkts = 0;
+		internals->rxq_scg[i].rx_bytes = 0;
+		internals->rxq_scg[i].err_pkts = 0;
+	}
+
+	internals->rx_missed = 0;
+
+	/* Tx */
+	for (i = 0; i < internals->nb_tx_queues; i++) {
+		internals->txq_scg[i].tx_pkts = 0;
+		internals->txq_scg[i].tx_bytes = 0;
+		internals->txq_scg[i].err_pkts = 0;
+	}
+
+	p_nt4ga_stat->n_totals_reset_timestamp = time(NULL);
+
+	pthread_mutex_unlock(&p_nt_drv->stat_lck);
+
+	return 0;
+}
+
 static int
 eth_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete __rte_unused)
 {
@@ -194,6 +292,23 @@ eth_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete __rte_unused)
 	return 0;
 }
 
+static int eth_stats_get(struct rte_eth_dev *eth_dev, struct rte_eth_stats *stats)
+{
+	struct pmd_internals *internals = (struct pmd_internals *)eth_dev->data->dev_private;
+	dpdk_stats_collect(internals, stats);
+	return 0;
+}
+
+static int eth_stats_reset(struct rte_eth_dev *eth_dev)
+{
+	struct pmd_internals *internals = (struct pmd_internals *)eth_dev->data->dev_private;
+	struct drv_s *p_drv = internals->p_drv;
+	struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+	const int if_index = internals->n_intf_no;
+	dpdk_stats_reset(internals, p_nt_drv, if_index);
+	return 0;
+}
+
 static int
 eth_dev_infos_get(struct rte_eth_dev *eth_dev, struct rte_eth_dev_info *dev_info)
 {
@@ -1455,6 +1570,8 @@ static const struct eth_dev_ops nthw_eth_dev_ops = {
 	.dev_set_link_down = eth_dev_set_link_down,
 	.dev_close = eth_dev_close,
 	.link_update = eth_link_update,
+	.stats_get = eth_stats_get,
+	.stats_reset = eth_stats_reset,
 	.dev_infos_get = eth_dev_infos_get,
 	.fw_version_get = eth_fw_version_get,
 	.rx_queue_setup = eth_rx_scg_queue_setup,
diff --git a/drivers/net/ntnic/ntnic_filter/ntnic_filter.c b/drivers/net/ntnic/ntnic_filter/ntnic_filter.c
index 4962ab8d5a..e2fce02afa 100644
--- a/drivers/net/ntnic/ntnic_filter/ntnic_filter.c
+++ b/drivers/net/ntnic/ntnic_filter/ntnic_filter.c
@@ -8,11 +8,19 @@
 #include "create_elements.h"
 #include "ntnic_mod_reg.h"
 #include "ntos_system.h"
+#include "ntos_drv.h"
 
 #define MAX_RTE_FLOWS 8192
 
+#define MAX_COLOR_FLOW_STATS 0x400
 #define NT_MAX_COLOR_FLOW_STATS 0x400
 
+#if (MAX_COLOR_FLOW_STATS != NT_MAX_COLOR_FLOW_STATS)
+#error Difference in COLOR_FLOW_STATS. Please synchronize the defines.
+#endif
+
+static struct rte_flow nt_flows[MAX_RTE_FLOWS];
+
 rte_spinlock_t flow_lock = RTE_SPINLOCK_INITIALIZER;
 static struct rte_flow nt_flows[MAX_RTE_FLOWS];
 
@@ -668,6 +676,9 @@ static int eth_flow_flush(struct rte_eth_dev *eth_dev, struct rte_flow_error *er
 				/* Cleanup recorded flows */
 				nt_flows[flow].used = 0;
 				nt_flows[flow].caller_id = 0;
+				nt_flows[flow].stat_bytes = 0UL;
+				nt_flows[flow].stat_pkts = 0UL;
+				nt_flows[flow].stat_tcp_flags = 0;
 			}
 		}
 
@@ -707,6 +718,127 @@ static int eth_flow_dev_dump(struct rte_eth_dev *eth_dev,
 	return res;
 }
 
+static int poll_statistics(struct pmd_internals *internals)
+{
+	int flow;
+	struct drv_s *p_drv = internals->p_drv;
+	struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+	nt4ga_stat_t *p_nt4ga_stat = &p_nt_drv->adapter_info.nt4ga_stat;
+	const int if_index = internals->n_intf_no;
+	uint64_t last_stat_rtc = 0;
+
+	if (!p_nt4ga_stat || if_index < 0 || if_index > NUM_ADAPTER_PORTS_MAX)
+		return -1;
+
+	assert(rte_tsc_freq > 0);
+
+	rte_spinlock_lock(&hwlock);
+
+	uint64_t now_rtc = rte_get_tsc_cycles();
+
+	/*
+	 * Check per port max once a second
+	 * if more than a second since last stat read, do a new one
+	 */
+	if ((now_rtc - internals->last_stat_rtc) < rte_tsc_freq) {
+		rte_spinlock_unlock(&hwlock);
+		return 0;
+	}
+
+	internals->last_stat_rtc = now_rtc;
+
+	pthread_mutex_lock(&p_nt_drv->stat_lck);
+
+	/*
+	 * Add the RX statistics increments since last time we polled.
+	 * (No difference if physical or virtual port)
+	 */
+	internals->rxq_scg[0].rx_pkts += p_nt4ga_stat->a_port_rx_packets_total[if_index] -
+		p_nt4ga_stat->a_port_rx_packets_base[if_index];
+	internals->rxq_scg[0].rx_bytes += p_nt4ga_stat->a_port_rx_octets_total[if_index] -
+		p_nt4ga_stat->a_port_rx_octets_base[if_index];
+	internals->rxq_scg[0].err_pkts += 0;
+	internals->rx_missed += p_nt4ga_stat->a_port_rx_drops_total[if_index] -
+		p_nt4ga_stat->a_port_rx_drops_base[if_index];
+
+	/* Update the increment bases */
+	p_nt4ga_stat->a_port_rx_packets_base[if_index] =
+		p_nt4ga_stat->a_port_rx_packets_total[if_index];
+	p_nt4ga_stat->a_port_rx_octets_base[if_index] =
+		p_nt4ga_stat->a_port_rx_octets_total[if_index];
+	p_nt4ga_stat->a_port_rx_drops_base[if_index] =
+		p_nt4ga_stat->a_port_rx_drops_total[if_index];
+
+	/* Tx (here we must distinguish between physical and virtual ports) */
+	if (internals->type == PORT_TYPE_PHYSICAL) {
+		/* Add the statistics increments since last time we polled */
+		internals->txq_scg[0].tx_pkts += p_nt4ga_stat->a_port_tx_packets_total[if_index] -
+			p_nt4ga_stat->a_port_tx_packets_base[if_index];
+		internals->txq_scg[0].tx_bytes += p_nt4ga_stat->a_port_tx_octets_total[if_index] -
+			p_nt4ga_stat->a_port_tx_octets_base[if_index];
+		internals->txq_scg[0].err_pkts += 0;
+
+		/* Update the increment bases */
+		p_nt4ga_stat->a_port_tx_packets_base[if_index] =
+			p_nt4ga_stat->a_port_tx_packets_total[if_index];
+		p_nt4ga_stat->a_port_tx_octets_base[if_index] =
+			p_nt4ga_stat->a_port_tx_octets_total[if_index];
+	}
+
+	/* Globally only once a second */
+	if ((now_rtc - last_stat_rtc) < rte_tsc_freq) {
+		rte_spinlock_unlock(&hwlock);
+		pthread_mutex_unlock(&p_nt_drv->stat_lck);
+		return 0;
+	}
+
+	last_stat_rtc = now_rtc;
+
+	/* All color counter are global, therefore only 1 pmd must update them */
+	const struct color_counters *p_color_counters = p_nt4ga_stat->mp_stat_structs_color;
+	struct color_counters *p_color_counters_base = p_nt4ga_stat->a_stat_structs_color_base;
+	uint64_t color_packets_accumulated, color_bytes_accumulated;
+
+	for (flow = 0; flow < MAX_RTE_FLOWS; flow++) {
+		if (nt_flows[flow].used) {
+			unsigned int color = nt_flows[flow].flow_stat_id;
+
+			if (color < NT_MAX_COLOR_FLOW_STATS) {
+				color_packets_accumulated = p_color_counters[color].color_packets;
+				nt_flows[flow].stat_pkts +=
+					(color_packets_accumulated -
+						p_color_counters_base[color].color_packets);
+
+				nt_flows[flow].stat_tcp_flags |= p_color_counters[color].tcp_flags;
+
+				color_bytes_accumulated = p_color_counters[color].color_bytes;
+				nt_flows[flow].stat_bytes +=
+					(color_bytes_accumulated -
+						p_color_counters_base[color].color_bytes);
+
+				/* Update the counter bases */
+				p_color_counters_base[color].color_packets =
+					color_packets_accumulated;
+				p_color_counters_base[color].color_bytes = color_bytes_accumulated;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&hwlock);
+	pthread_mutex_unlock(&p_nt_drv->stat_lck);
+
+	return 0;
+}
+
+static const struct ntnic_filter_ops ntnic_filter_ops = {
+	.poll_statistics = poll_statistics,
+};
+
+void ntnic_filter_init(void)
+{
+	register_ntnic_filter_ops(&ntnic_filter_ops);
+}
+
 static const struct rte_flow_ops dev_flow_ops = {
 	.create = eth_flow_create,
 	.destroy = eth_flow_destroy,
diff --git a/drivers/net/ntnic/ntnic_mod_reg.c b/drivers/net/ntnic/ntnic_mod_reg.c
index 593b56bf5b..355e2032b1 100644
--- a/drivers/net/ntnic/ntnic_mod_reg.c
+++ b/drivers/net/ntnic/ntnic_mod_reg.c
@@ -19,6 +19,21 @@ const struct sg_ops_s *get_sg_ops(void)
 	return sg_ops;
 }
 
+static const struct ntnic_filter_ops *ntnic_filter_ops;
+
+void register_ntnic_filter_ops(const struct ntnic_filter_ops *ops)
+{
+	ntnic_filter_ops = ops;
+}
+
+const struct ntnic_filter_ops *get_ntnic_filter_ops(void)
+{
+	if (ntnic_filter_ops == NULL)
+		ntnic_filter_init();
+
+	return ntnic_filter_ops;
+}
+
 static struct link_ops_s *link_100g_ops;
 
 void register_100g_link_ops(struct link_ops_s *ops)
@@ -47,6 +62,21 @@ const struct port_ops *get_port_ops(void)
 	return port_ops;
 }
 
+static const struct nt4ga_stat_ops *nt4ga_stat_ops;
+
+void register_nt4ga_stat_ops(const struct nt4ga_stat_ops *ops)
+{
+	nt4ga_stat_ops = ops;
+}
+
+const struct nt4ga_stat_ops *get_nt4ga_stat_ops(void)
+{
+	if (nt4ga_stat_ops == NULL)
+		nt4ga_stat_ops_init();
+
+	return nt4ga_stat_ops;
+}
+
 static const struct adapter_ops *adapter_ops;
 
 void register_adapter_ops(const struct adapter_ops *ops)
diff --git a/drivers/net/ntnic/ntnic_mod_reg.h b/drivers/net/ntnic/ntnic_mod_reg.h
index e40ed9b949..30b9afb7d3 100644
--- a/drivers/net/ntnic/ntnic_mod_reg.h
+++ b/drivers/net/ntnic/ntnic_mod_reg.h
@@ -111,6 +111,14 @@ void register_sg_ops(struct sg_ops_s *ops);
 const struct sg_ops_s *get_sg_ops(void);
 void sg_init(void);
 
+struct ntnic_filter_ops {
+	int (*poll_statistics)(struct pmd_internals *internals);
+};
+
+void register_ntnic_filter_ops(const struct ntnic_filter_ops *ops);
+const struct ntnic_filter_ops *get_ntnic_filter_ops(void);
+void ntnic_filter_init(void);
+
 struct link_ops_s {
 	int (*link_init)(struct adapter_info_s *p_adapter_info, nthw_fpga_t *p_fpga);
 };
@@ -175,6 +183,15 @@ void register_port_ops(const struct port_ops *ops);
 const struct port_ops *get_port_ops(void);
 void port_init(void);
 
+struct nt4ga_stat_ops {
+	int (*nt4ga_stat_init)(struct adapter_info_s *p_adapter_info);
+	int (*nt4ga_stat_setup)(struct adapter_info_s *p_adapter_info);
+};
+
+void register_nt4ga_stat_ops(const struct nt4ga_stat_ops *ops);
+const struct nt4ga_stat_ops *get_nt4ga_stat_ops(void);
+void nt4ga_stat_ops_init(void);
+
 struct adapter_ops {
 	int (*init)(struct adapter_info_s *p_adapter_info);
 	int (*deinit)(struct adapter_info_s *p_adapter_info);
diff --git a/drivers/net/ntnic/ntutil/nt_util.h b/drivers/net/ntnic/ntutil/nt_util.h
index a482fb43ad..f2eccf3501 100644
--- a/drivers/net/ntnic/ntutil/nt_util.h
+++ b/drivers/net/ntnic/ntutil/nt_util.h
@@ -22,6 +22,7 @@
  * The windows size must max be 3 min in order to
  * prevent overflow.
  */
+#define PORT_LOAD_WINDOWS_SIZE 2ULL
 #define FLM_LOAD_WINDOWS_SIZE 2ULL
 
 #define PCIIDENT_TO_DOMAIN(pci_ident) ((uint16_t)(((unsigned int)(pci_ident) >> 16) & 0xFFFFU))
-- 
2.45.0


  parent reply	other threads:[~2024-10-22 17:30 UTC|newest]

Thread overview: 229+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2024-10-21 21:04 [PATCH v1 00/73] Provide flow filter API and statistics Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 01/73] net/ntnic: add API for configuration NT flow dev Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 02/73] net/ntnic: add flow filter API Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 03/73] net/ntnic: add minimal create/destroy flow operations Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 04/73] net/ntnic: add internal flow create/destroy API Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 05/73] net/ntnic: add minimal NT flow inline profile Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 06/73] net/ntnic: add management API for NT flow profile Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 07/73] net/ntnic: add NT flow profile management implementation Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 08/73] net/ntnic: add create/destroy implementation for NT flows Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 09/73] net/ntnic: add infrastructure for for flow actions and items Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 10/73] net/ntnic: add action queue Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 11/73] net/ntnic: add action mark Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 12/73] net/ntnic: add ation jump Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 13/73] net/ntnic: add action drop Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 14/73] net/ntnic: add item eth Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 15/73] net/ntnic: add item IPv4 Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 16/73] net/ntnic: add item ICMP Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 17/73] net/ntnic: add item port ID Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 18/73] net/ntnic: add item void Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 19/73] net/ntnic: add item UDP Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 20/73] net/ntnic: add action TCP Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 21/73] net/ntnic: add action VLAN Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 22/73] net/ntnic: add item SCTP Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 23/73] net/ntnic: add items IPv6 and ICMPv6 Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 24/73] net/ntnic: add action modify filed Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 25/73] net/ntnic: add items gtp and actions raw encap/decap Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 26/73] net/ntnic: add cat module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 27/73] net/ntnic: add SLC LR module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 28/73] net/ntnic: add PDB module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 29/73] net/ntnic: add QSL module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 30/73] net/ntnic: add KM module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 31/73] net/ntnic: add hash API Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 32/73] net/ntnic: add TPE module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 33/73] net/ntnic: add FLM module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 34/73] net/ntnic: add flm rcp module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 35/73] net/ntnic: add learn flow queue handling Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 36/73] net/ntnic: match and action db attributes were added Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 37/73] net/ntnic: add flow dump feature Serhii Iliushyk
2024-10-21 23:10   ` Stephen Hemminger
2024-10-21 21:04 ` [PATCH v1 38/73] net/ntnic: add flow flush Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 39/73] net/ntnic: add GMF (Generic MAC Feeder) module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 40/73] net/ntnic: sort FPGA registers alphanumerically Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 41/73] net/ntnic: add MOD CSU Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 42/73] net/ntnic: add MOD FLM Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 43/73] net/ntnic: add HFU module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 44/73] net/ntnic: add IFR module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 45/73] net/ntnic: add MAC Rx module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 46/73] net/ntnic: add MAC Tx module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 47/73] net/ntnic: add RPP LR module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 48/73] net/ntnic: add MOD SLC LR Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 49/73] net/ntnic: add Tx CPY module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 50/73] net/ntnic: add Tx INS module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 51/73] net/ntnic: add Tx RPL module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 52/73] net/ntnic: update alignment for virt queue structs Serhii Iliushyk
2024-10-21 23:12   ` Stephen Hemminger
2024-10-21 21:04 ` [PATCH v1 53/73] net/ntnic: enable RSS feature Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 54/73] net/ntnic: add statistics API Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 55/73] net/ntnic: add rpf module Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 56/73] net/ntnic: add statistics poll Serhii Iliushyk
2024-10-21 21:04 ` [PATCH v1 57/73] net/ntnic: added flm stat interface Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 58/73] net/ntnic: add tsm module Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 59/73] net/ntnic: add STA module Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 60/73] net/ntnic: add TSM module Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 61/73] net/ntnic: add xstats Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 62/73] net/ntnic: added flow statistics Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 63/73] net/ntnic: add scrub registers Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 64/73] net/ntnic: update documentation Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 65/73] net/ntnic: added flow aged APIs Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 66/73] net/ntnic: add aged API to the inline profile Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 67/73] net/ntnic: add info and configure flow API Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 68/73] net/ntnic: add aged flow event Serhii Iliushyk
2024-10-21 23:22   ` Stephen Hemminger
2024-10-21 21:05 ` [PATCH v1 69/73] net/ntnic: add thread termination Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 70/73] net/ntnic: add age documentation Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 71/73] net/ntnic: add meter API Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 72/73] net/ntnic: add meter module Serhii Iliushyk
2024-10-21 21:05 ` [PATCH v1 73/73] net/ntnic: add meter documentation Serhii Iliushyk
2024-10-22 16:54 ` [PATCH v2 00/73] Provide flow filter API and statistics Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 01/73] net/ntnic: add API for configuration NT flow dev Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 02/73] net/ntnic: add flow filter API Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 03/73] net/ntnic: add minimal create/destroy flow operations Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 04/73] net/ntnic: add internal flow create/destroy API Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 05/73] net/ntnic: add minimal NT flow inline profile Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 06/73] net/ntnic: add management API for NT flow profile Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 07/73] net/ntnic: add NT flow profile management implementation Serhii Iliushyk
2024-10-22 17:17     ` Stephen Hemminger
2024-10-22 16:54   ` [PATCH v2 08/73] net/ntnic: add create/destroy implementation for NT flows Serhii Iliushyk
2024-10-22 17:20     ` Stephen Hemminger
2024-10-23 16:09       ` Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 09/73] net/ntnic: add infrastructure for for flow actions and items Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 10/73] net/ntnic: add action queue Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 11/73] net/ntnic: add action mark Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 12/73] net/ntnic: add ation jump Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 13/73] net/ntnic: add action drop Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 14/73] net/ntnic: add item eth Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 15/73] net/ntnic: add item IPv4 Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 16/73] net/ntnic: add item ICMP Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 17/73] net/ntnic: add item port ID Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 18/73] net/ntnic: add item void Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 19/73] net/ntnic: add item UDP Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 20/73] net/ntnic: add action TCP Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 21/73] net/ntnic: add action VLAN Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 22/73] net/ntnic: add item SCTP Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 23/73] net/ntnic: add items IPv6 and ICMPv6 Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 24/73] net/ntnic: add action modify filed Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 25/73] net/ntnic: add items gtp and actions raw encap/decap Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 26/73] net/ntnic: add cat module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 27/73] net/ntnic: add SLC LR module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 28/73] net/ntnic: add PDB module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 29/73] net/ntnic: add QSL module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 30/73] net/ntnic: add KM module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 31/73] net/ntnic: add hash API Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 32/73] net/ntnic: add TPE module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 33/73] net/ntnic: add FLM module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 34/73] net/ntnic: add flm rcp module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 35/73] net/ntnic: add learn flow queue handling Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 36/73] net/ntnic: match and action db attributes were added Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 37/73] net/ntnic: add flow dump feature Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 38/73] net/ntnic: add flow flush Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 39/73] net/ntnic: add GMF (Generic MAC Feeder) module Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 40/73] net/ntnic: sort FPGA registers alphanumerically Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 41/73] net/ntnic: add MOD CSU Serhii Iliushyk
2024-10-22 16:54   ` [PATCH v2 42/73] net/ntnic: add MOD FLM Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 43/73] net/ntnic: add HFU module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 44/73] net/ntnic: add IFR module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 45/73] net/ntnic: add MAC Rx module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 46/73] net/ntnic: add MAC Tx module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 47/73] net/ntnic: add RPP LR module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 48/73] net/ntnic: add MOD SLC LR Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 49/73] net/ntnic: add Tx CPY module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 50/73] net/ntnic: add Tx INS module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 51/73] net/ntnic: add Tx RPL module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 52/73] net/ntnic: update alignment for virt queue structs Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 53/73] net/ntnic: enable RSS feature Serhii Iliushyk
2024-10-22 16:55   ` Serhii Iliushyk [this message]
2024-10-22 16:55   ` [PATCH v2 55/73] net/ntnic: add rpf module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 56/73] net/ntnic: add statistics poll Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 57/73] net/ntnic: added flm stat interface Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 58/73] net/ntnic: add tsm module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 59/73] net/ntnic: add STA module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 60/73] net/ntnic: add TSM module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 61/73] net/ntnic: add xstats Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 62/73] net/ntnic: added flow statistics Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 63/73] net/ntnic: add scrub registers Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 64/73] net/ntnic: update documentation Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 65/73] net/ntnic: added flow aged APIs Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 66/73] net/ntnic: add aged API to the inline profile Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 67/73] net/ntnic: add info and configure flow API Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 68/73] net/ntnic: add aged flow event Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 69/73] net/ntnic: add thread termination Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 70/73] net/ntnic: add age documentation Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 71/73] net/ntnic: add meter API Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 72/73] net/ntnic: add meter module Serhii Iliushyk
2024-10-22 16:55   ` [PATCH v2 73/73] net/ntnic: add meter documentation Serhii Iliushyk
2024-10-22 17:11   ` [PATCH v2 00/73] Provide flow filter API and statistics Stephen Hemminger
2024-10-23 16:59 ` [PATCH v3 " Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 01/73] net/ntnic: add API for configuration NT flow dev Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 02/73] net/ntnic: add flow filter API Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 03/73] net/ntnic: add minimal create/destroy flow operations Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 04/73] net/ntnic: add internal flow create/destroy API Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 05/73] net/ntnic: add minimal NT flow inline profile Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 06/73] net/ntnic: add management API for NT flow profile Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 07/73] net/ntnic: add NT flow profile management implementation Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 08/73] net/ntnic: add create/destroy implementation for NT flows Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 09/73] net/ntnic: add infrastructure for for flow actions and items Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 10/73] net/ntnic: add action queue Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 11/73] net/ntnic: add action mark Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 12/73] net/ntnic: add ation jump Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 13/73] net/ntnic: add action drop Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 14/73] net/ntnic: add item eth Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 15/73] net/ntnic: add item IPv4 Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 16/73] net/ntnic: add item ICMP Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 17/73] net/ntnic: add item port ID Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 18/73] net/ntnic: add item void Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 19/73] net/ntnic: add item UDP Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 20/73] net/ntnic: add action TCP Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 21/73] net/ntnic: add action VLAN Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 22/73] net/ntnic: add item SCTP Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 23/73] net/ntnic: add items IPv6 and ICMPv6 Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 24/73] net/ntnic: add action modify filed Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 25/73] net/ntnic: add items gtp and actions raw encap/decap Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 26/73] net/ntnic: add cat module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 27/73] net/ntnic: add SLC LR module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 28/73] net/ntnic: add PDB module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 29/73] net/ntnic: add QSL module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 30/73] net/ntnic: add KM module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 31/73] net/ntnic: add hash API Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 32/73] net/ntnic: add TPE module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 33/73] net/ntnic: add FLM module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 34/73] net/ntnic: add flm rcp module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 35/73] net/ntnic: add learn flow queue handling Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 36/73] net/ntnic: match and action db attributes were added Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 37/73] net/ntnic: add flow dump feature Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 38/73] net/ntnic: add flow flush Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 39/73] net/ntnic: add GMF (Generic MAC Feeder) module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 40/73] net/ntnic: sort FPGA registers alphanumerically Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 41/73] net/ntnic: add MOD CSU Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 42/73] net/ntnic: add MOD FLM Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 43/73] net/ntnic: add HFU module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 44/73] net/ntnic: add IFR module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 45/73] net/ntnic: add MAC Rx module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 46/73] net/ntnic: add MAC Tx module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 47/73] net/ntnic: add RPP LR module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 48/73] net/ntnic: add MOD SLC LR Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 49/73] net/ntnic: add Tx CPY module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 50/73] net/ntnic: add Tx INS module Serhii Iliushyk
2024-10-23 16:59   ` [PATCH v3 51/73] net/ntnic: add Tx RPL module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 52/73] net/ntnic: update alignment for virt queue structs Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 53/73] net/ntnic: enable RSS feature Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 54/73] net/ntnic: add statistics API Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 55/73] net/ntnic: add rpf module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 56/73] net/ntnic: add statistics poll Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 57/73] net/ntnic: added flm stat interface Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 58/73] net/ntnic: add tsm module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 59/73] net/ntnic: add STA module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 60/73] net/ntnic: add TSM module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 61/73] net/ntnic: add xstats Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 62/73] net/ntnic: added flow statistics Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 63/73] net/ntnic: add scrub registers Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 64/73] net/ntnic: update documentation Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 65/73] net/ntnic: add flow aging API Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 66/73] net/ntnic: add aging API to the inline profile Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 67/73] net/ntnic: add flow info and flow configure APIs Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 68/73] net/ntnic: add flow aging event Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 69/73] net/ntnic: add termination thread Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 70/73] net/ntnic: add aging documentation Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 71/73] net/ntnic: add meter API Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 72/73] net/ntnic: add meter module Serhii Iliushyk
2024-10-23 17:00   ` [PATCH v3 73/73] net/ntnic: update meter documentation Serhii Iliushyk

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20241022165541.3186140-55-sil-plv@napatech.com \
    --to=sil-plv@napatech.com \
    --cc=andrew.rybchenko@oktetlabs.ru \
    --cc=ckm@napatech.com \
    --cc=dev@dpdk.org \
    --cc=dvo-plv@napatech.com \
    --cc=ferruh.yigit@amd.com \
    --cc=mko-plv@napatech.com \
    --cc=stephen@networkplumber.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).