DPDK patches and discussions
 help / color / mirror / Atom feed
From: Chaoyong He <chaoyong.he@corigine.com>
To: dev@dpdk.org
Cc: oss-drivers@corigine.com, Long Wu <long.wu@corigine.com>,
	Chaoyong He <chaoyong.he@corigine.com>,
	Peng Zhang <peng.zhang@corigine.com>
Subject: [PATCH v2 3/3] net/nfp: flower driver uses one service core
Date: Fri,  2 Feb 2024 11:04:15 +0800	[thread overview]
Message-ID: <20240202030415.739638-4-chaoyong.he@corigine.com> (raw)
In-Reply-To: <20240202030415.739638-1-chaoyong.he@corigine.com>

From: Long Wu <long.wu@corigine.com>

Now app must provide a service core for each NFP pci device that uses
flower driver to run flower service, which will cause huge resource waste.
This patch is to solve this problem and app can only use one core to
run NFP flower service though it uses several NFP NICs.

Signed-off-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/net/nfp/flower/nfp_flower.c           |  47 ++---
 drivers/net/nfp/flower/nfp_flower_ctrl.c      |  20 +-
 drivers/net/nfp/flower/nfp_flower_ctrl.h      |   2 +-
 .../net/nfp/flower/nfp_flower_representor.c   |   4 +
 drivers/net/nfp/flower/nfp_flower_service.c   | 196 ++++++++++++++++++
 drivers/net/nfp/flower/nfp_flower_service.h   |  17 ++
 drivers/net/nfp/meson.build                   |   1 +
 drivers/net/nfp/nfp_net_common.h              |   6 +
 drivers/net/nfp/nfpcore/nfp_sync.h            |   2 +
 9 files changed, 248 insertions(+), 47 deletions(-)
 create mode 100644 drivers/net/nfp/flower/nfp_flower_service.c
 create mode 100644 drivers/net/nfp/flower/nfp_flower_service.h

diff --git a/drivers/net/nfp/flower/nfp_flower.c b/drivers/net/nfp/flower/nfp_flower.c
index a29edf8ca2..e84e6ebbff 100644
--- a/drivers/net/nfp/flower/nfp_flower.c
+++ b/drivers/net/nfp/flower/nfp_flower.c
@@ -18,6 +18,7 @@
 #include "../nfp_mtr.h"
 #include "nfp_flower_ctrl.h"
 #include "nfp_flower_representor.h"
+#include "nfp_flower_service.h"
 
 #define CTRL_VNIC_NB_DESC 512
 
@@ -461,6 +462,13 @@ nfp_flower_init_ctrl_vnic(struct nfp_net_hw *hw)
 		nn_cfg_writeb(&hw->super, NFP_NET_CFG_TXR_SZ(i), rte_log2_u32(CTRL_VNIC_NB_DESC));
 	}
 
+	/* Alloc sync memory zone */
+	ret = nfp_flower_service_sync_alloc(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Alloc sync memory zone failed");
+		goto tx_queue_setup_cleanup;
+	}
+
 	return 0;
 
 tx_queue_setup_cleanup:
@@ -531,6 +539,7 @@ nfp_flower_cleanup_ctrl_vnic(struct nfp_net_hw *hw)
 		}
 	}
 
+	nfp_flower_service_sync_free(app_fw_flower);
 	rte_free(eth_dev->data->tx_queues);
 	rte_free(eth_dev->data->rx_queues);
 	rte_mempool_free(app_fw_flower->ctrl_pktmbuf_pool);
@@ -584,38 +593,6 @@ nfp_flower_start_ctrl_vnic(struct nfp_net_hw *net_hw)
 	return 0;
 }
 
-static int
-nfp_flower_ctrl_vnic_service(void *arg)
-{
-	struct nfp_app_fw_flower *app_fw_flower = arg;
-
-	nfp_flower_ctrl_vnic_poll(app_fw_flower);
-
-	return 0;
-}
-
-static int
-nfp_flower_enable_services(struct nfp_app_fw_flower *app_fw_flower)
-{
-	int ret;
-	struct nfp_service_info info;
-	const struct rte_service_spec flower_service = {
-		.name              = "flower_ctrl_vnic_service",
-		.callback          = nfp_flower_ctrl_vnic_service,
-		.callback_userdata = (void *)app_fw_flower,
-	};
-
-	ret = nfp_service_enable(&flower_service, &info);
-	if (ret != 0) {
-		PMD_INIT_LOG(ERR, "Could not enable service %s", flower_service.name);
-		return ret;
-	}
-
-	app_fw_flower->ctrl_vnic_id = info.id;
-
-	return 0;
-}
-
 static void
 nfp_flower_pkt_add_metadata_register(struct nfp_app_fw_flower *app_fw_flower)
 {
@@ -760,7 +737,7 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 	}
 
 	/* Start up flower services */
-	ret = nfp_flower_enable_services(app_fw_flower);
+	ret = nfp_flower_service_start(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not enable flower services");
 		ret = -ESRCH;
@@ -770,11 +747,13 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
 	ret = nfp_flower_repr_create(app_fw_flower);
 	if (ret != 0) {
 		PMD_INIT_LOG(ERR, "Could not create representor ports");
-		goto ctrl_vnic_cleanup;
+		goto ctrl_vnic_service_stop;
 	}
 
 	return 0;
 
+ctrl_vnic_service_stop:
+	nfp_flower_service_stop(app_fw_flower);
 ctrl_vnic_cleanup:
 	nfp_flower_cleanup_ctrl_vnic(app_fw_flower->ctrl_hw);
 ctrl_cpp_area_cleanup:
diff --git a/drivers/net/nfp/flower/nfp_flower_ctrl.c b/drivers/net/nfp/flower/nfp_flower_ctrl.c
index c25487c277..bcb325d475 100644
--- a/drivers/net/nfp/flower/nfp_flower_ctrl.c
+++ b/drivers/net/nfp/flower/nfp_flower_ctrl.c
@@ -12,6 +12,7 @@
 #include "../nfp_logs.h"
 #include "nfp_flower_representor.h"
 #include "nfp_mtr.h"
+#include "nfp_flower_service.h"
 
 #define MAX_PKT_BURST 32
 
@@ -502,26 +503,21 @@ nfp_flower_cmsg_rx(struct nfp_app_fw_flower *app_fw_flower,
 }
 
 void
-nfp_flower_ctrl_vnic_poll(struct nfp_app_fw_flower *app_fw_flower)
+nfp_flower_ctrl_vnic_process(struct nfp_app_fw_flower *app_fw_flower)
 {
 	uint16_t count;
 	struct nfp_net_rxq *rxq;
-	struct nfp_net_hw *ctrl_hw;
 	struct rte_eth_dev *ctrl_eth_dev;
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 
-	ctrl_hw = app_fw_flower->ctrl_hw;
-	ctrl_eth_dev = ctrl_hw->eth_dev;
+	ctrl_eth_dev = app_fw_flower->ctrl_hw->eth_dev;
 
 	/* Ctrl vNIC only has a single Rx queue */
 	rxq = ctrl_eth_dev->data->rx_queues[0];
-
-	while (rte_service_runstate_get(app_fw_flower->ctrl_vnic_id) != 0) {
-		count = nfp_flower_ctrl_vnic_recv(rxq, pkts_burst, MAX_PKT_BURST);
-		if (count != 0) {
-			app_fw_flower->ctrl_vnic_rx_count += count;
-			/* Process cmsgs here */
-			nfp_flower_cmsg_rx(app_fw_flower, pkts_burst, count);
-		}
+	count = nfp_flower_ctrl_vnic_recv(rxq, pkts_burst, MAX_PKT_BURST);
+	if (count != 0) {
+		app_fw_flower->ctrl_vnic_rx_count += count;
+		/* Process cmsgs here */
+		nfp_flower_cmsg_rx(app_fw_flower, pkts_burst, count);
 	}
 }
diff --git a/drivers/net/nfp/flower/nfp_flower_ctrl.h b/drivers/net/nfp/flower/nfp_flower_ctrl.h
index 4c94d36847..131d002ac6 100644
--- a/drivers/net/nfp/flower/nfp_flower_ctrl.h
+++ b/drivers/net/nfp/flower/nfp_flower_ctrl.h
@@ -8,7 +8,7 @@
 
 #include "nfp_flower.h"
 
-void nfp_flower_ctrl_vnic_poll(struct nfp_app_fw_flower *app_fw_flower);
+void nfp_flower_ctrl_vnic_process(struct nfp_app_fw_flower *app_fw_flower);
 uint16_t nfp_flower_ctrl_vnic_xmit(struct nfp_app_fw_flower *app_fw_flower,
 		struct rte_mbuf *mbuf);
 void nfp_flower_ctrl_vnic_xmit_register(struct nfp_app_fw_flower *app_fw_flower);
diff --git a/drivers/net/nfp/flower/nfp_flower_representor.c b/drivers/net/nfp/flower/nfp_flower_representor.c
index 12e8af54fb..4f4df0cd2e 100644
--- a/drivers/net/nfp/flower/nfp_flower_representor.c
+++ b/drivers/net/nfp/flower/nfp_flower_representor.c
@@ -9,6 +9,7 @@
 #include "../nfpcore/nfp_nsp.h"
 #include "../nfp_logs.h"
 #include "../nfp_mtr.h"
+#include "nfp_flower_service.h"
 
 /* Type of representor */
 enum nfp_repr_type {
@@ -396,6 +397,9 @@ nfp_flower_repr_dev_close(struct rte_eth_dev *dev)
 	if (app_fw_flower->pf_repr != NULL)
 		return 0;
 
+	/* Stop flower service first */
+	nfp_flower_service_stop(app_fw_flower);
+
 	/* Disable cpp service */
 	nfp_service_disable(&pf_dev->cpp_service_info);
 
diff --git a/drivers/net/nfp/flower/nfp_flower_service.c b/drivers/net/nfp/flower/nfp_flower_service.c
new file mode 100644
index 0000000000..5e8811fe24
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_service.c
@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2024 Corigine, Inc.
+ * All rights reserved.
+ */
+
+#include "nfp_flower_service.h"
+
+#include <rte_spinlock.h>
+
+#include "nfp_flower_ctrl.h"
+#include "../nfpcore/nfp_cpp.h"
+#include "../nfpcore/nfp_sync.h"
+#include "../nfp_logs.h"
+#include "../nfp_service.h"
+
+/* Driver limitation, PMD can enlarge it if need. */
+#define MAX_FLOWER_SERVICE_SLOT 8
+
+struct nfp_flower_service {
+	/** Flower service is enabled */
+	bool enabled;
+	/** Flower service info */
+	struct nfp_service_info info;
+	/** Store flower cards' information */
+	struct nfp_app_fw_flower *slots[MAX_FLOWER_SERVICE_SLOT];
+	/** Spinlock for sync slots when add/remove card */
+	rte_spinlock_t spinlock;
+};
+
+static struct nfp_flower_service *
+nfp_flower_service_handle_get(struct nfp_app_fw_flower *app)
+{
+	return app->pf_hw->pf_dev->process_share.fl_service;
+}
+
+static int
+nfp_flower_service_loop(void *arg)
+{
+	uint16_t slot;
+	struct nfp_app_fw_flower *app;
+	struct nfp_flower_service *service_handle;
+
+	service_handle = arg;
+	/* Waiting for enabling service */
+	while (!service_handle->enabled)
+		rte_delay_ms(1);
+
+	while (rte_service_runstate_get(service_handle->info.id) != 0) {
+		rte_spinlock_lock(&service_handle->spinlock);
+		for (slot = 0; slot < MAX_FLOWER_SERVICE_SLOT; slot++) {
+			app = service_handle->slots[slot];
+			if (app == NULL)
+				continue;
+
+			nfp_flower_ctrl_vnic_process(app);
+		}
+		rte_spinlock_unlock(&service_handle->spinlock);
+	}
+
+	return 0;
+}
+
+static int
+nfp_flower_service_enable(struct nfp_flower_service *service_handle)
+{
+	int ret;
+
+	const struct rte_service_spec flower_service = {
+		.name              = "flower_ctrl_vnic_service",
+		.callback          = nfp_flower_service_loop,
+		.callback_userdata = (void *)service_handle,
+	};
+
+	ret = nfp_service_enable(&flower_service, &service_handle->info);
+	if (ret != 0)
+		return ret;
+
+	rte_spinlock_init(&service_handle->spinlock);
+	service_handle->enabled = true;
+
+	return 0;
+}
+
+static uint16_t
+nfp_flower_service_insert(struct nfp_app_fw_flower *app,
+		struct nfp_flower_service *service_handle)
+{
+	uint16_t slot;
+
+	rte_spinlock_lock(&service_handle->spinlock);
+	for (slot = 0; slot < MAX_FLOWER_SERVICE_SLOT; slot++) {
+		if (service_handle->slots[slot] == NULL) {
+			service_handle->slots[slot] = app;
+			break;
+		}
+	}
+	rte_spinlock_unlock(&service_handle->spinlock);
+
+	return slot;
+}
+
+int
+nfp_flower_service_start(void *app_fw_flower)
+{
+	int ret;
+	struct nfp_flower_service *service_handle;
+	struct nfp_app_fw_flower *app = app_fw_flower;
+
+	service_handle = nfp_flower_service_handle_get(app);
+	if (service_handle == NULL) {
+		PMD_DRV_LOG(ERR, "Can not get service handle");
+		return -EINVAL;
+	}
+
+	/* Enable flower service when driver initializes the first NIC */
+	if (!service_handle->enabled) {
+		ret = nfp_flower_service_enable(service_handle);
+		if (ret != 0) {
+			PMD_DRV_LOG(ERR, "Could not enable flower service");
+			return -ESRCH;
+		}
+	}
+
+	/* Insert the NIC to flower service slot */
+	ret = nfp_flower_service_insert(app, service_handle);
+	if (ret == MAX_FLOWER_SERVICE_SLOT) {
+		PMD_DRV_LOG(ERR, "Flower ctrl vnic service slot over %u",
+				MAX_FLOWER_SERVICE_SLOT);
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+void
+nfp_flower_service_stop(void *app_fw_flower)
+{
+	uint16_t slot;
+	uint16_t count;
+	struct nfp_flower_service *service_handle;
+	struct nfp_app_fw_flower *app = app_fw_flower;
+
+	service_handle = nfp_flower_service_handle_get(app);
+	if (service_handle == NULL) {
+		PMD_DRV_LOG(ERR, "Can not get service handle");
+		return;
+	}
+
+	rte_spinlock_lock(&service_handle->spinlock);
+	for (slot = 0; slot < MAX_FLOWER_SERVICE_SLOT; slot++) {
+		/* The app only in one slot */
+		if (service_handle->slots[slot] != app)
+			continue;
+
+		service_handle->slots[slot] = NULL;
+	}
+	rte_spinlock_unlock(&service_handle->spinlock);
+
+	/* Determine whether to disable service */
+	count = nfp_sync_handle_count_get(app->pf_hw->pf_dev->sync, NULL,
+			service_handle);
+	if (count > 1)
+		return;
+
+	if (nfp_service_disable(&service_handle->info) != 0)
+		PMD_DRV_LOG(ERR, "Could not disable service");
+}
+
+int
+nfp_flower_service_sync_alloc(void *app_fw_flower)
+{
+	struct nfp_flower_service *service_handle;
+	struct nfp_app_fw_flower *app = app_fw_flower;
+	struct nfp_pf_dev *pf_dev = app->pf_hw->pf_dev;
+
+	service_handle = nfp_sync_handle_alloc(pf_dev->sync, NULL,
+			NFP_SYNC_MAGIC_FL_SERVICE,
+			sizeof(struct nfp_flower_service));
+	if (service_handle == NULL)
+		return -ENOMEM;
+
+	pf_dev->process_share.fl_service = service_handle;
+
+	return 0;
+}
+
+void
+nfp_flower_service_sync_free(void *app_fw_flower)
+{
+	struct nfp_app_fw_flower *app = app_fw_flower;
+	struct nfp_pf_dev *pf_dev = app->pf_hw->pf_dev;
+
+	nfp_sync_handle_free(pf_dev->sync, NULL, pf_dev->process_share.fl_service);
+
+	pf_dev->process_share.fl_service = NULL;
+}
diff --git a/drivers/net/nfp/flower/nfp_flower_service.h b/drivers/net/nfp/flower/nfp_flower_service.h
new file mode 100644
index 0000000000..a75780274f
--- /dev/null
+++ b/drivers/net/nfp/flower/nfp_flower_service.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2024 Corigine, Inc.
+ * All rights reserved.
+ */
+
+#ifndef __NFP_FLOWER_SERVICE_H__
+#define __NFP_FLOWER_SERVICE_H__
+
+struct nfp_flower_service;
+
+int nfp_flower_service_start(void *app_fw_flower);
+void nfp_flower_service_stop(void *app_fw_flower);
+
+int nfp_flower_service_sync_alloc(void *app_fw_flower);
+void nfp_flower_service_sync_free(void *app_fw_flower);
+
+#endif /* __NFP_FLOWER_SERVICE_H__ */
diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build
index 0f4beccbdf..e376fd328f 100644
--- a/drivers/net/nfp/meson.build
+++ b/drivers/net/nfp/meson.build
@@ -13,6 +13,7 @@ sources = files(
         'flower/nfp_flower_ctrl.c',
         'flower/nfp_flower_flow.c',
         'flower/nfp_flower_representor.c',
+        'flower/nfp_flower_service.c',
         'nfd3/nfp_nfd3_dp.c',
         'nfdk/nfp_nfdk_dp.c',
         'nfpcore/nfp_cppcore.c',
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index eaa7501e8e..e374739022 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -12,6 +12,7 @@
 #include <nfp_dev.h>
 #include <rte_spinlock.h>
 
+#include "flower/nfp_flower_service.h"
 #include "nfpcore/nfp_sync.h"
 #include "nfp_net_ctrl.h"
 #include "nfp_service.h"
@@ -80,6 +81,10 @@ struct nfp_multi_pf {
 	uint8_t *beat_addr;
 };
 
+struct nfp_process_share {
+	struct nfp_flower_service *fl_service;
+};
+
 struct nfp_pf_dev {
 	/** Backpointer to associated pci device */
 	struct rte_pci_device *pci_dev;
@@ -114,6 +119,7 @@ struct nfp_pf_dev {
 
 	/** Synchronized info */
 	struct nfp_sync *sync;
+	struct nfp_process_share process_share;
 };
 
 #define NFP_NET_FLOW_LIMIT    1024
diff --git a/drivers/net/nfp/nfpcore/nfp_sync.h b/drivers/net/nfp/nfpcore/nfp_sync.h
index 82f01e2652..ff3f60937e 100644
--- a/drivers/net/nfp/nfpcore/nfp_sync.h
+++ b/drivers/net/nfp/nfpcore/nfp_sync.h
@@ -10,6 +10,8 @@
 
 #include <bus_pci_driver.h>
 
+#define NFP_SYNC_MAGIC_FL_SERVICE            0x53594e41 /**< ASCII - SYNA */
+
 struct nfp_sync;
 
 struct nfp_sync *nfp_sync_alloc(void);
-- 
2.39.1


  parent reply	other threads:[~2024-02-02  3:05 UTC|newest]

Thread overview: 14+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2024-01-24  9:25 [PATCH 0/4] enhance NFP service framework Chaoyong He
2024-01-24  9:25 ` [PATCH 1/4] net/nfp: add synchronize module Chaoyong He
2024-01-24  9:25 ` [PATCH 2/4] net/nfp: create new service code related files Chaoyong He
2024-01-24  9:25 ` [PATCH 3/4] net/nfp: flower driver uses one service core Chaoyong He
2024-01-24  9:25 ` [PATCH 4/4] net/nfp: fix stop cpp service problem Chaoyong He
2024-01-30 12:07   ` Ferruh Yigit
2024-01-30 12:17     ` Kevin Traynor
2024-02-02  3:04 ` [PATCH v2 0/3] enhance NFP service framework Chaoyong He
2024-02-02  3:04   ` [PATCH v2 1/3] net/nfp: add synchronize module Chaoyong He
2024-02-02  3:04   ` [PATCH v2 2/3] net/nfp: create new service code related files Chaoyong He
2024-02-02  3:04   ` Chaoyong He [this message]
2024-02-06 15:17   ` [PATCH v2 0/3] enhance NFP service framework Ferruh Yigit
2024-02-08  9:52     ` Chaoyong He
2024-02-08 13:58   ` Ferruh Yigit

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20240202030415.739638-4-chaoyong.he@corigine.com \
    --to=chaoyong.he@corigine.com \
    --cc=dev@dpdk.org \
    --cc=long.wu@corigine.com \
    --cc=oss-drivers@corigine.com \
    --cc=peng.zhang@corigine.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).