[PATCH] ethdev: add template table resize API

[PATCH] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handlers held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update all existing table flow rules by calling
  `rte_flow_async_update_resized()`.
  The table resize procedure does not change application flow handler.
  However, flow object can reference internal PMD resources that are
  obsolete after table resize.
  `rte_flow_async_update_resized()` moves internal flow references
  to the updated table resources.
  The flow update must not interrupt hardware flow operations.

* When all table flow were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>

---
 app/test-pmd/cmdline_flow.c            |  86 +++++++++++++++++++--
 app/test-pmd/config.c                  | 102 +++++++++++++++++++++++++
 app/test-pmd/testpmd.h                 |   6 ++
 doc/guides/rel_notes/release_24_03.rst |   2 +
 lib/ethdev/ethdev_trace.h              |  33 ++++++++
 lib/ethdev/ethdev_trace_points.c       |   9 +++
 lib/ethdev/rte_flow.c                  |  69 +++++++++++++++++
 lib/ethdev/rte_flow.h                  |  97 +++++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h           |  15 ++++
 lib/ethdev/version.map                 |   3 +
 10 files changed, 417 insertions(+), 5 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..a34757a13e 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index e9c9717706..8ad1891396 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,8 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for template API table resize.**
+
 
 Removed Items
 -------------
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index 549e329558..3a46a81f44 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2479,3 +2479,72 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index affdc8121b..0b5de9b20b 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5749,6 +5749,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5827,6 +5831,13 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+static __rte_always_inline bool
+rte_flow_table_resizable(const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
+}
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6753,6 +6764,92 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Change template table flow rules capacity.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* cannot be resized or invalid *nb_rules*
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, update flow resources in port.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *rule* cannot be updated.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, notify port that all table flows were updated.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) PMD cannot complete table resize.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..e37bab9b81 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,9 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+	rte_flow_template_table_resize;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH] ethdev: add template table resize API

[Ferruh Yigit]

On 12/17/2023 9:32 AM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
> RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handlers held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update all existing table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The table resize procedure does not change application flow handler.
>   However, flow object can reference internal PMD resources that are
>   obsolete after table resize.
>   `rte_flow_async_update_resized()` moves internal flow references
>   to the updated table resources.
>   The flow update must not interrupt hardware flow operations.
> 
> * When all table flow were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 

Hi Gregory, Ori,

Why we need three separate APIs,
 rte_flow_template_table_resize
 rte_flow_async_update_resized
 rte_flow_template_table_resize_complete

Why not 'rte_flow_template_table_resize()' update existing flows and
release resources related to the original tables automatically?

Re: [PATCH] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,

>
> Hi Gregory, Ori,
>
> Why we need three separate APIs,
> rte_flow_template_table_resize
> rte_flow_async_update_resized
> rte_flow_template_table_resize_complete
>
> Why not 'rte_flow_template_table_resize()' update existing flows and
> release resources related to the original tables automatically?
>


Template table resize API allows to add new flows immediately after 
rte_flow_template_table_resize completed.
A multi-threaded application can add new and update old flows simultaneously.

A single resize-and-update API would require to lock PMD for entire operation.
For application with 1e6 flows doubling a table would end up with 
considerable down time.

The rte_flow_template_table_resize_complete was added for PMDs that cannot 
differentiate flows created before and after table resize.

Regards,
Gregory

Re: [PATCH] ethdev: add template table resize API

[Ferruh Yigit]

On 1/29/2024 3:08 PM, Etelson, Gregory wrote:
> Hello Ferruh,
> 
>>
>> Hi Gregory, Ori,
>>
>> Why we need three separate APIs,
>> rte_flow_template_table_resize
>> rte_flow_async_update_resized
>> rte_flow_template_table_resize_complete
>>
>> Why not 'rte_flow_template_table_resize()' update existing flows and
>> release resources related to the original tables automatically?
>>
> 
> 
> Template table resize API allows to add new flows immediately after
> rte_flow_template_table_resize completed.
> A multi-threaded application can add new and update old flows
> simultaneously.
> 
> A single resize-and-update API would require to lock PMD for entire
> operation.
> For application with 1e6 flows doubling a table would end up with
> considerable down time.
> 

If a multi-threaded application can add new and updated old
simultaneously, this should be done via monolithic API, like:
{
  lock
    resize
  unlock
  for each flow
    lock
    update
    unlock
}

Perhaps questions is, is there a usecase that user does the resize but
doesn't want to update the old flows?

> The rte_flow_template_table_resize_complete was added for PMDs that
> cannot differentiate flows created before and after table resize.
> 

Can you please elaborate this?

Does it mean old flows and new flows require different handling and PMD
doesn't know how to differentiate old and new flows?
If so how update() converts old flows, there must be a way for driver to
differentiate them for update() to work.

Also if resize_complete() NOT called at all, does this mean PMD can't
handle any flows anymore as it can't differentiate old and new ones?

Re: [PATCH] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,

>
> If a multi-threaded application can add new and updated old
> simultaneously, this should be done via monolithic API, like:
> {
>  lock
>    resize
>  unlock
>  for each flow
>    lock
>    update
>    unlock
> }
>

The flow template API was designed for performance.
Application that implements the flow template API expects high
flows insertions, updates, and removals rates.
Locks are necessary for the table resize API.

During the monolithic resize, application has no control over
PMD locks. Even if resize and each update operations are
relatively fast, application should expect table lock collisions
in rules insertions, deletions and updates for the entire 
resize-and-update-and-update operation.

With the separate resize API, lock collisions are expected
during the resize phase only.
After table resize completed, all flow operations will obtain
a lock without collision.
Also, application does not have to update all flows at once.
Updates can be executed in batches scheduled by application.
Another use case: application can increase a table,
offload all new flows and run updates while hardware handles
network traffic according to the new flows scheme.

> Perhaps questions is, is there a usecase that user does the resize but
> doesn't want to update the old flows?

Please see below.

>
>> The rte_flow_template_table_resize_complete was added for PMDs that
>> cannot differentiate flows created before and after table resize.
>>
>
> Can you please elaborate this?
>
> Does it mean old flows and new flows require different handling and PMD
> doesn't know how to differentiate old and new flows?
> If so how update() converts old flows, there must be a way for driver to
> differentiate them for update() to work.
>
> Also if resize_complete() NOT called at all, does this mean PMD can't
> handle any flows anymore as it can't differentiate old and new ones?
>

Table resize API do not have any effect on running flows.
PMD uses the same procedure to create flows before and after table resize.
All flows instantiated from the same type before and after table resize.

Flow update that follows table resize manages PMD flow object location.
In MLX5 PMD, flow update moves an object that references a flow
from old table to a new table.
After all flows were moved to a new table, PMD has no need for the old
table and it can be released.

Since flow update manages PMD memory only,
application can ignore the update operation
if it does not care about effective memory management.

PMD can release the old flow table after all flows it referenced were
moved to a new table only.
Event that notifies PMD about empty old table can be ether internal
or external.
Internal event assumes PMD ability to track flows in a table.
External event in form of application call is more general approach.

Application must call resize_complete after it moved all flows
to the new table. That call notifies PMD that it safe to release resources 
related to old table.
If application did not update flows it must not call resize_complte.

Application can create new flows after table resize
regardless if it managed PMD memory with update and
resize_complete calls or not.

Re: [PATCH] ethdev: add template table resize API

[Ferruh Yigit]

On 1/30/2024 12:46 PM, Etelson, Gregory wrote:
> Hello Ferruh,
> 
>>
>> If a multi-threaded application can add new and updated old
>> simultaneously, this should be done via monolithic API, like:
>> {
>>  lock
>>    resize
>>  unlock
>>  for each flow
>>    lock
>>    update
>>    unlock
>> }
>>
> 
> The flow template API was designed for performance.
> Application that implements the flow template API expects high
> flows insertions, updates, and removals rates.
> Locks are necessary for the table resize API.
> 
> During the monolithic resize, application has no control over
> PMD locks. Even if resize and each update operations are
> relatively fast, application should expect table lock collisions
> in rules insertions, deletions and updates for the entire
> resize-and-update-and-update operation.
> 
> With the separate resize API, lock collisions are expected
> during the resize phase only.
> After table resize completed, all flow operations will obtain
> a lock without collision.
> Also, application does not have to update all flows at once.
> Updates can be executed in batches scheduled by application.
> Another use case: application can increase a table,
> offload all new flows and run updates while hardware handles
> network traffic according to the new flows scheme.
> 
>> Perhaps questions is, is there a usecase that user does the resize but
>> doesn't want to update the old flows?
> 
> Please see below.
> 
>>
>>> The rte_flow_template_table_resize_complete was added for PMDs that
>>> cannot differentiate flows created before and after table resize.
>>>
>>
>> Can you please elaborate this?
>>
>> Does it mean old flows and new flows require different handling and PMD
>> doesn't know how to differentiate old and new flows?
>> If so how update() converts old flows, there must be a way for driver to
>> differentiate them for update() to work.
>>
>> Also if resize_complete() NOT called at all, does this mean PMD can't
>> handle any flows anymore as it can't differentiate old and new ones?
>>
> 
> Table resize API do not have any effect on running flows.
> PMD uses the same procedure to create flows before and after table resize.
> All flows instantiated from the same type before and after table resize.
> 
> Flow update that follows table resize manages PMD flow object location.
> In MLX5 PMD, flow update moves an object that references a flow
> from old table to a new table.
> After all flows were moved to a new table, PMD has no need for the old
> table and it can be released.
> 
> Since flow update manages PMD memory only,
> application can ignore the update operation
> if it does not care about effective memory management.
> 
> PMD can release the old flow table after all flows it referenced were
> moved to a new table only.
> Event that notifies PMD about empty old table can be ether internal
> or external.
> Internal event assumes PMD ability to track flows in a table.
> External event in form of application call is more general approach.
> 
> Application must call resize_complete after it moved all flows
> to the new table. That call notifies PMD that it safe to release
> resources related to old table.
> If application did not update flows it must not call resize_complte.
> 
> Application can create new flows after table resize
> regardless if it managed PMD memory with update and
> resize_complete calls or not.
>

Thanks for clarification.

So, by design, driver will keep the old table when it is resized.
- Can this have a performance impact, like when rules
updated/removed/inserted driver will need to look more tables?
- Or can this cause additional capacity complexity, like total number of
rules will be sum of rules in all tables, but new rules only can be
added to latest table, so number of rules can be more than size of
latest table.
- Or user can add more flows after resize() and this may not leave
enough room to update old rules to new table, what is expected behavior
for this case?
- Or if user did not updated rules at all after resize(), after each
rule deletion driver won't need to check if old table emptied and needs
to be freed?
- Or can user call resize() API multiple times, causing driver to
maintain multiple tables? How much memory overhead this may bring?


'rte_flow_async_update_resized()' API is called per flow, won't this
force application to trace which flows are created in new table and
which are in old table, so pushing additional work to application.

Or what will happen if update() fails in the middle of update, should
user retry, should PMD restore back the moved rules?



I understood the logic behind the dividing responsibility to multiple
APIs, and it makes sense, but it brings above complexities, and more
work to application.
Can it be possible to have monolithic API but only resize() part of it
is blocking and update() part and later remove table part done
asynchronously?


I will also put more comment on the patch based on latest understanding.

Re: [PATCH] ethdev: add template table resize API

[Ferruh Yigit]

On 12/17/2023 9:32 AM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
> RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handlers held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update all existing table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The table resize procedure does not change application flow handler.
>   However, flow object can reference internal PMD resources that are
>   obsolete after table resize.
>   `rte_flow_async_update_resized()` moves internal flow references
>   to the updated table resources.
>   The flow update must not interrupt hardware flow operations.
> 
> * When all table flow were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 
> Signed-off-by: Gregory Etelson <getelson@nvidia.com>
> Acked-by: Ori Kam <orika@nvidia.com>
> 
> ---
>  app/test-pmd/cmdline_flow.c            |  86 +++++++++++++++++++--
>  app/test-pmd/config.c                  | 102 +++++++++++++++++++++++++
>  app/test-pmd/testpmd.h                 |   6 ++
>

It helps to document added/updated command in the commit log, and in the
documentation 'testpmd_funcs.rst'.
Including new attribute to create table etc...

Please check following commit as sample:
Commit 98ab16778daa ("app/testpmd: add random item support")


>  doc/guides/rel_notes/release_24_03.rst |   2 +
>  lib/ethdev/ethdev_trace.h              |  33 ++++++++
>  lib/ethdev/ethdev_trace_points.c       |   9 +++
>  lib/ethdev/rte_flow.c                  |  69 +++++++++++++++++
>  lib/ethdev/rte_flow.h                  |  97 +++++++++++++++++++++++
>  lib/ethdev/rte_flow_driver.h           |  15 ++++
>  lib/ethdev/version.map                 |   3 +
>  10 files changed, 417 insertions(+), 5 deletions(-)
> 

Just to double check if there will be driver implementation of the new
APIs in this release?

<...>

>  
> +static __rte_always_inline bool
> +rte_flow_table_resizable(const struct rte_flow_template_table_attr *tbl_attr)
> +{
> +	return (tbl_attr->specialize &
> +		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
> +}
> +

Ahh, this is the 4th API added in this patch, missed before.

Why it is exposed to the application?
What is the expected usage for the function, it doesn't get 'port_id' as
parameter and directly works on the table_attribute, feels like API
should be for drivers?
Why it is inlined?

<...>

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Change template table flow rules capacity.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table to modify.
> + * @param nb_rules
> + *   New flow rules capacity.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *table* cannot be resized or invalid *nb_rules*
>

What is invalid 'nb_rules', does it make sense to document it in above
@param for it. Like if 0 valid, or is there a max, or is it allowed to
shrink the size of table?


> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize(uint16_t port_id,
> +			       struct rte_flow_template_table *table,
> +			       uint32_t nb_rules,
> +			       struct rte_flow_error *error);
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, update flow resources in port.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param queue
> + *   Flow queue for async operation.
> + * @param attr
> + *   Async operation attributes.
> + * @param rule
> + *   Flow rule to update.
> + * @param user_data
> + *   The user data that will be returned on async completion event.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *rule* cannot be updated.
> + */
> +__rte_experimental
> +int
> +rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
> +			      const struct rte_flow_op_attr *attr,
> +			      struct rte_flow *rule, void *user_data,
> +			      struct rte_flow_error *error);
> +

Should API have 'table' or 'template' in name, if it is out of this
context, it is not intuitive that update_resized refers to template
table resize. It may be confused as if this is something related to the
rule itself resize.


> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, notify port that all table flows were updated.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table that undergoing resize operation.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) PMD cannot complete table resize.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize_complete(uint16_t port_id,
> +					struct rte_flow_template_table *table,
> +					struct rte_flow_error *error);
>

Does it make sense to add a new error type to differentiate
unrecoverable error (I don't know if there is any) and an error that is
caused by not all rules updated?

<...>

>  
>  /**
> diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
> index 5c4917c020..e37bab9b81 100644
> --- a/lib/ethdev/version.map
> +++ b/lib/ethdev/version.map
> @@ -316,6 +316,9 @@ EXPERIMENTAL {
>  	rte_eth_recycle_rx_queue_info_get;
>  	rte_flow_group_set_miss_actions;
>  	rte_flow_calc_table_hash;
> +	rte_flow_template_table_resize;
> +	rte_flow_async_update_resized;
> +	rte_flow_template_table_resize_complete;
>  

New APIs should go below "# added in 24.03" comment

Re: [PATCH] ethdev: add template table resize API

[Etelson, Gregory]

[-- Attachment #1: Type: text/plain, Size: 3001 bytes --]

Hello Ferruh,

> So, by design, driver will keep the old table when it is resized.
> - Can this have a performance impact, like when rules
> updated/removed/inserted driver will need to look more tables?
> - Or can this cause additional capacity complexity, like total number of
> rules will be sum of rules in all tables, but new rules only can be
> added to latest table, so number of rules can be more than size of
> latest table.
> - Or user can add more flows after resize() and this may not leave
> enough room to update old rules to new table, what is expected behavior
> for this case?
> - Or if user did not updated rules at all after resize(), after each
> rule deletion driver won't need to check if old table emptied and needs
> to be freed?
> - Or can user call resize() API multiple times, causing driver to
> maintain multiple tables? How much memory overhead this may bring?

After "resize, update, complete" sequence table performance must be
the same as before resize.
If application skiped updates or resize completion, performance is undefined.
Driver must verify that total flows number does not exceed capacity set in 
table resize.

> 'rte_flow_async_update_resized()' API is called per flow, won't this
> force application to trace which flows are created in new table and
> which are in old table, so pushing additional work to application.

Application must trace what flows require update after table resize.
As the general rule, all flows that were created before table resize call has 
returned must be updated:

"old" flows |<-----------------resize------------>| "new" flows
   update                                              keep
                unknown flow location: update

----------------------------TIME-------------------------------------->

In MLX5 PMD, if update was called with a "new" flow, the call returns will 
success, without changing the flow.

>
> Or what will happen if update() fails in the middle of update, should
> user retry, should PMD restore back the moved rules?
>

If flow update call failed, it treated as failure during flow create, update or 
destroy.

>
> I understood the logic behind the dividing responsibility to multiple
> APIs, and it makes sense, but it brings above complexities, and more
> work to application.
> Can it be possible to have monolithic API but only resize() part of it
> is blocking and update() part and later remove table part done
> asynchronously?
>

Table resize and single flow update operations consume approximately the same 
time duration.
An update of a table with 1_000_000 flows will consume driver for too much time.
During that time application will not be able to create, destroy or update 
existing "old" flows.
Such operation must be coordinated with application.

A driver could provide a batch flows update, but I don’t see how it helps.
It's ether update one or update all and the latter does not scale.

>
> I will also put more comment on the patch based on latest understanding.
>
>

Re: [PATCH] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,

>> Template table creation API sets table flows capacity.
>> If application needs more flows then the table was designed for,
>> the following procedures must be completed:
>> 1. Create a new template table with larger flows capacity.
>> 2. Re-create existing flows in the new table and delete flows from
>>    the original table.
>> 3. Destroy original table.
>>
>> Application cannot always execute that procedure:
>> * Port may not have sufficient resources to allocate a new table
>>   while maintaining original table.
>> * Application may not have existing flows "recipes" to re-create
>>   flows in a new table.
>>
>> The patch defines a new API that allows application to resize
>> existing template table:
>>
>> * Resizable template table must be created with the
>> RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
>>
>> * Application resizes existing table with the
>>   `rte_flow_template_table_resize()` function call.
>>   The table resize procedure updates the table maximal flow number
>>   only. Other table attributes are not affected by the table resize.
>>   ** The table resize procedure must not interrupt
>>      existing table flows operations in hardware.
>>   ** The table resize procedure must not alter flow handlers held by
>>      application.
>>
>> * After `rte_flow_template_table_resize()` returned, application must
>>   update all existing table flow rules by calling
>>   `rte_flow_async_update_resized()`.
>>   The table resize procedure does not change application flow handler.
>>   However, flow object can reference internal PMD resources that are
>>   obsolete after table resize.
>>   `rte_flow_async_update_resized()` moves internal flow references
>>   to the updated table resources.
>>   The flow update must not interrupt hardware flow operations.
>>
>> * When all table flow were updated, application must call
>>   `rte_flow_template_table_resize_complete()`.
>>   The function releases PMD resources related to the original
>>   table.
>>   Application can start new table resize after
>>   `rte_flow_template_table_resize_complete()` returned.
>>
>> Signed-off-by: Gregory Etelson <getelson@nvidia.com>
>> Acked-by: Ori Kam <orika@nvidia.com>
>>
>> ---
>>  app/test-pmd/cmdline_flow.c            |  86 +++++++++++++++++++--
>>  app/test-pmd/config.c                  | 102 +++++++++++++++++++++++++
>>  app/test-pmd/testpmd.h                 |   6 ++
>>
>
> It helps to document added/updated command in the commit log, and in the
> documentation 'testpmd_funcs.rst'.
> Including new attribute to create table etc...
>
> Please check following commit as sample:
> Commit 98ab16778daa ("app/testpmd: add random item support")
>

I'll post v2 with an update.

>
>>  doc/guides/rel_notes/release_24_03.rst |   2 +
>>  lib/ethdev/ethdev_trace.h              |  33 ++++++++
>>  lib/ethdev/ethdev_trace_points.c       |   9 +++
>>  lib/ethdev/rte_flow.c                  |  69 +++++++++++++++++
>>  lib/ethdev/rte_flow.h                  |  97 +++++++++++++++++++++++
>>  lib/ethdev/rte_flow_driver.h           |  15 ++++
>>  lib/ethdev/version.map                 |   3 +
>>  10 files changed, 417 insertions(+), 5 deletions(-)
>>
>
> Just to double check if there will be driver implementation of the new
> APIs in this release?
>

I'll post driver patches after DPDK API approval.

> <...>
>
>>
>> +static __rte_always_inline bool
>> +rte_flow_table_resizable(const struct rte_flow_template_table_attr *tbl_attr)
>> +{
>> +     return (tbl_attr->specialize &
>> +             RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
>> +}
>> +
>
> Ahh, this is the 4th API added in this patch, missed before.
>
> Why it is exposed to the application?

That API is for application and a driver.

> What is the expected usage for the function, it doesn't get 'port_id' as
> parameter and directly works on the table_attribute, feels like API

There is no usage for `port_id` in that query.

> should be for drivers?
> Why it is inlined?

I prefer inline functions to macros.

>
> <...>
>
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice.
>> + *
>> + * Change template table flow rules capacity.
>> + *
>> + * @param port_id
>> + *   Port identifier of Ethernet device.
>> + * @param table
>> + *   Template table to modify.
>> + * @param nb_rules
>> + *   New flow rules capacity.
>> + * @param error
>> + *   Perform verbose error reporting if not NULL.
>> + *   PMDs initialize this structure in case of error only.
>> + *
>> + * @return
>> + *   - (0) if success.
>> + *   - (-ENODEV) if *port_id* invalid.
>> + *   - (-ENOTSUP) if underlying device does not support this functionality.
>> + *   - (-EINVAL) if *table* cannot be resized or invalid *nb_rules*
>>
>
> What is invalid 'nb_rules', does it make sense to document it in above
> @param for it. Like if 0 valid, or is there a max, or is it allowed to
> shrink the size of table?

I'll post v2 with an update.

>
>
>> + */
>> +__rte_experimental
>> +int
>> +rte_flow_template_table_resize(uint16_t port_id,
>> +                            struct rte_flow_template_table *table,
>> +                            uint32_t nb_rules,
>> +                            struct rte_flow_error *error);
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice.
>> + *
>> + * Following table resize, update flow resources in port.
>> + *
>> + * @param port_id
>> + *   Port identifier of Ethernet device.
>> + * @param queue
>> + *   Flow queue for async operation.
>> + * @param attr
>> + *   Async operation attributes.
>> + * @param rule
>> + *   Flow rule to update.
>> + * @param user_data
>> + *   The user data that will be returned on async completion event.
>> + * @param error
>> + *   Perform verbose error reporting if not NULL.
>> + *   PMDs initialize this structure in case of error only.
>> + *
>> + * @return
>> + *   - (0) if success.
>> + *   - (-ENODEV) if *port_id* invalid.
>> + *   - (-ENOTSUP) if underlying device does not support this functionality.
>> + *   - (-EINVAL) if *rule* cannot be updated.
>> + */
>> +__rte_experimental
>> +int
>> +rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
>> +                           const struct rte_flow_op_attr *attr,
>> +                           struct rte_flow *rule, void *user_data,
>> +                           struct rte_flow_error *error);
>> +
>
> Should API have 'table' or 'template' in name, if it is out of this
> context, it is not intuitive that update_resized refers to template
> table resize. It may be confused as if this is something related to the
> rule itself resize.

That function call should have been called
`rte_flow_async_update_flow_after_table_resize`
The current version was selected after a long debate.
Please suggest an alternative.

>
>
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice.
>> + *
>> + * Following table resize, notify port that all table flows were updated.
>> + *
>> + * @param port_id
>> + *   Port identifier of Ethernet device.
>> + * @param table
>> + *   Template table that undergoing resize operation.
>> + * @param error
>> + *   Perform verbose error reporting if not NULL.
>> + *   PMDs initialize this structure in case of error only.
>> + *
>> + * @return
>> + *   - (0) if success.
>> + *   - (-ENODEV) if *port_id* invalid.
>> + *   - (-ENOTSUP) if underlying device does not support this functionality.
>> + *   - (-EINVAL) PMD cannot complete table resize.
>> + */
>> +__rte_experimental
>> +int
>> +rte_flow_template_table_resize_complete(uint16_t port_id,
>> +                                     struct rte_flow_template_table *table,
>> +                                     struct rte_flow_error *error);
>>
>
> Does it make sense to add a new error type to differentiate
> unrecoverable error (I don't know if there is any) and an error that is
> caused by not all rules updated?

That API should not return unrecoverable error.
When it called, all hardware related updates already completed.
No new resources required.

>
> <...>
>
>>
>>  /**
>> diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
>> index 5c4917c020..e37bab9b81 100644
>> --- a/lib/ethdev/version.map
>> +++ b/lib/ethdev/version.map
>> @@ -316,6 +316,9 @@ EXPERIMENTAL {
>>       rte_eth_recycle_rx_queue_info_get;
>>       rte_flow_group_set_miss_actions;
>>       rte_flow_calc_table_hash;
>> +     rte_flow_template_table_resize;
>> +     rte_flow_async_update_resized;
>> +     rte_flow_template_table_resize_complete;
>>
>
> New APIs should go below "# added in 24.03" comment
>
I'll post an update in v2.

[PATCH v2] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handlers held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update all existing table flow rules by calling
  `rte_flow_async_update_resized()`.
  The table resize procedure does not change application flow handler.
  However, flow object can reference internal PMD resources that are
  obsolete after table resize.
  `rte_flow_async_update_resized()` moves internal flow references
  to the updated table resources.
  The flow update must not interrupt hardware flow operations.

* When all table flow were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
---
 app/test-pmd/cmdline_flow.c                 |  86 ++++++++++++++-
 app/test-pmd/config.c                       | 102 ++++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 ++
 doc/guides/rel_notes/release_24_03.rst      |   2 +
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  69 ++++++++++++
 lib/ethdev/rte_flow.h                       | 114 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   5 +
 11 files changed, 450 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..a34757a13e 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 6f8ad27808..047664a079 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,8 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for template API table resize.**
+
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index f49d1d3767..4f8320ec96 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2481,3 +2481,72 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..338f06454c 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,28 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query if a table can be resized
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table
+ *
+ * @return
+ *   True if the table can be resized.
+ */
+static __rte_always_inline bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
+}
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6776,94 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Change template table flow rules capacity.
+ * PMD implementation must support table change to the new size.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* cannot be resized or resize to *nb_rules*
+ *               is not supported in PMD.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, update flow resources in port.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *rule* cannot be updated.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, notify port that all table flows were updated.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) PMD cannot complete table resize.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..bae70eff27 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,11 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_template_table_resize;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v2] ethdev: add template table resize API

[Thomas Monjalon]

31/01/2024 10:59, Gregory Etelson:
> --- a/lib/ethdev/rte_flow.h
> +++ b/lib/ethdev/rte_flow.h
>  #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
> +/**
> + * Specialize table for resize.
> + */
> +#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE RTE_BIT32(2)

I'm not sure about the repeating "TABLE" at the end of this flag name.

[...]
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Query if a table can be resized

s/if/whether/
A dot is missing.

> + *
> + * @param port_id
> + *    Port identifier of Ethernet device.
> + * @param tbl_attr
> + *    Template table

Every lines should end with a dot in Doxygen for consistency.

> + *
> + * @return
> + *   True if the table can be resized.
> + */
> +static __rte_always_inline bool

Why is it inline?
In general we avoid inline except for few performance sensitive ones.

> +rte_flow_table_resizable(__rte_unused uint16_t port_id,
> +			 const struct rte_flow_template_table_attr *tbl_attr)
> +{
> +	return (tbl_attr->specialize &
> +		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
> +}

[...]
> +	# added in 24.03
> +	rte_flow_template_table_resize;
> +	rte_flow_async_update_resized;
> +	rte_flow_template_table_resize_complete;

I like the idea of these 3 new functions.
The scenario should be described in doc/guides/prog_guide/rte_flow.rst

[PATCH v3] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handlers held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update all existing table flow rules by calling
  `rte_flow_async_update_resized()`.
  The table resize procedure does not change application flow handler.
  However, flow object can reference internal PMD resources that are
  obsolete after table resize.
  `rte_flow_async_update_resized()` moves internal flow references
  to the updated table resources.
  The flow update must not interrupt hardware flow operations.

* When all table flow were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
---
 app/test-pmd/cmdline_flow.c                 |  86 ++++++++++++++-
 app/test-pmd/config.c                       | 102 ++++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 ++
 doc/guides/rel_notes/release_24_03.rst      |   2 +
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  69 ++++++++++++
 lib/ethdev/rte_flow.h                       | 114 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   5 +
 11 files changed, 450 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..1a2556d53b 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 6f8ad27808..047664a079 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,8 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for template API table resize.**
+
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index f49d1d3767..4f8320ec96 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2481,3 +2481,72 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..43f29ef0d9 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,28 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query if a table can be resized
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table
+ *
+ * @return
+ *   True if the table can be resized.
+ */
+static __rte_always_inline bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6776,94 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Change template table flow rules capacity.
+ * PMD implementation must support table change to the new size.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* cannot be resized or resize to *nb_rules*
+ *               is not supported in PMD.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, update flow resources in port.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *rule* cannot be updated.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, notify port that all table flows were updated.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) PMD cannot complete table resize.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..bae70eff27 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,11 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_template_table_resize;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v2] ethdev: add template table resize API

[Etelson, Gregory]

Hello Thomas,

>> --- a/lib/ethdev/rte_flow.h
>> +++ b/lib/ethdev/rte_flow.h
>>  #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
>> +/**
>> + * Specialize table for resize.
>> + */
>> +#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE RTE_BIT32(2)
>
> I'm not sure about the repeating "TABLE" at the end of this flag name.
>
> [...]
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice.
>> + *
>> + * Query if a table can be resized
>
> s/if/whether/
> A dot is missing.
>
>> + *
>> + * @param port_id
>> + *    Port identifier of Ethernet device.
>> + * @param tbl_attr
>> + *    Template table
>
> Every lines should end with a dot in Doxygen for consistency.
>
>> + *
>> + * @return
>> + *   True if the table can be resized.
>> + */
>> +static __rte_always_inline bool
>
> Why is it inline?
> In general we avoid inline except for few performance sensitive ones.
>
>> +rte_flow_table_resizable(__rte_unused uint16_t port_id,
>> +                      const struct rte_flow_template_table_attr *tbl_attr)
>> +{
>> +     return (tbl_attr->specialize &
>> +             RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE) != 0;
>> +}
>
> [...]
>> +     # added in 24.03
>> +     rte_flow_template_table_resize;
>> +     rte_flow_async_update_resized;
>> +     rte_flow_template_table_resize_complete;
>
> I like the idea of these 3 new functions.
> The scenario should be described in doc/guides/prog_guide/rte_flow.rst
>
>
>

I'll post an update.

[PATCH v4] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handlers held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update all existing table flow rules by calling
  `rte_flow_async_update_resized()`.
  The table resize procedure does not change application flow handler.
  However, flow object can reference internal PMD resources that are
  obsolete after table resize.
  `rte_flow_async_update_resized()` moves internal flow references
  to the updated table resources.
  The flow update must not interrupt hardware flow operations.

* When all table flow were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
v4: Remove inline.
    Add use case to rte_flow.rst.
---
 app/test-pmd/cmdline_flow.c                 |  86 ++++++++++++++-
 app/test-pmd/config.c                       | 102 ++++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 ++
 doc/guides/howto/rte_flow.rst               | 111 ++++++++++++++++++++
 doc/guides/rel_notes/release_24_03.rst      |   2 +
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  77 ++++++++++++++
 lib/ethdev/rte_flow.h                       | 111 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   6 ++
 12 files changed, 567 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..1a2556d53b 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/howto/rte_flow.rst b/doc/guides/howto/rte_flow.rst
index 27d4f28f77..2349e7450c 100644
--- a/doc/guides/howto/rte_flow.rst
+++ b/doc/guides/howto/rte_flow.rst
@@ -303,3 +303,114 @@ Terminal 1: output log::
   received packet with src ip = 176.80.50.4 sent to queue 3
   received packet with src ip = 176.80.50.5 sent to queue 1
   received packet with src ip = 176.80.50.6 sent to queue 3
+
+Template API resizable table
+----------------------------
+
+Description
+~~~~~~~~~~~
+
+This example shows how to work with resizable template table.
+
+The code is equivalent to the following testpmd commands:(wrapped for
+clarity)::
+
+  # 1. Create resizable template table for 1 flow.
+  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
+                template eth / ipv4 / udp src mask 0xffff / end
+  testpmd> flow actions_template 0 create ingress actions_template_id 7
+                template count  / rss / end
+  testpmd> flow template_table 0 create table_id 101 resizable ingress
+                group 1 priority 0 rules_number 1
+                pattern_template 3 actions_template 7
+
+  # 2. Queue a flow rule.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
+
+  # 3. Resize the template table
+  #    The new table capacity is 32 rules
+  testpmd> flow template_table 0 resize table_resize_id 101
+                table_resize_rules_num 32
+
+  # 4. Queue more flow rules.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
+
+  # 5. Queue the initial flow update.
+  testpmd> flow queue 0 update_resized 0 rule 0
+
+  #6. Complete the table resize.
+  flow template_table 0 resize_complete table 101
+
+Code
+~~~~
+
+.. code-block:: c
+
+  / * Create resizable table with initial capacity 1. */
+  const struct rte_flow_template_table_attr _table_attr_= {
+    .nb_flows = 1, /* Initial capacity. */
+    .specialize = RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE /* Can resize. */
+  };
+  struct rte_flow_template_table *_table_;
+
+  _table_ = rte_flow_template_table_create
+            (port_id,
+            _table_attr_,
+            pattern_templates, 1,
+            actions_templates, nb_actions_templates, error);
+
+  /* Queue flow rule 0. */
+  struct rte_flow *_flow_0_;
+  _flow_0_ = rte_flow_async_create
+              (port_id, queue_id, op_attr,
+              _table_,
+              pattern_0, pattern_template_index,
+              actions, actions_template_index,
+              user_data_0, error);
+
+  /* Resize the table. */
+  uint32_t _new_table_capacity_ = 32;
+  rte_flow_template_table_resize
+              (port_id,
+              _table_,
+              _new_table_capacity_,
+              error);
+
+  /* Create additional flows. */
+  struct rte_flow *_flow_1_, *_flow_2_, *_flow_3_;
+  _flow_1_ = rte_flow_async_create
+              (port_id, queue_id, op_attr,
+              _table_,
+              pattern_1, pattern_template_index,
+              actions, actions_template_index,
+              user_data_1, error);
+  _flow_2_ = rte_flow_async_create
+              (port_id, queue_id, op_attr,
+              _table_,
+              pattern_2, pattern_template_index,
+              actions, actions_template_index,
+              user_data_2, error);
+  _flow_3_ = rte_flow_async_create
+              (port_id, queue_id, op_attr,
+              _table_,
+              pattern_3, pattern_template_index,
+              actions, actions_template_index,
+              user_data_3, error);
+
+  /* Queue _flow_0_ update. */
+  rte_flow_async_update_resized(port_id, queue, attr,
+                                _flow_0_,
+                                user_data, error);
+
+  /* Complete table resize. */
+  rte_flow_template_table_resize_complete(port_id, _table_, error);
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 6f8ad27808..047664a079 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,8 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for template API table resize.**
+
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index f49d1d3767..99987e6518 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2481,3 +2481,80 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..9aedc06516 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,25 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query whether a table can be resized.
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table.
+ *
+ * @return
+ *   True if the table can be resized.
+ */
+__rte_experimental
+bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6773,94 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Change template table flow rules capacity.
+ * PMD implementation must support table change to the new size.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* cannot be resized or resize to *nb_rules*
+ *               is not supported in PMD.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, update flow resources in port.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *rule* cannot be updated.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, notify port that all table flows were updated.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) PMD cannot complete table resize.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..5b22aa598a 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,12 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_table_resizable;
+	rte_flow_template_table_resize;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH] ethdev: add template table resize API

[Ferruh Yigit]

On 1/30/2024 6:15 PM, Etelson, Gregory wrote:
> Hello Ferruh,
> 
>> So, by design, driver will keep the old table when it is resized.
>> - Can this have a performance impact, like when rules
>> updated/removed/inserted driver will need to look more tables?
>> - Or can this cause additional capacity complexity, like total number of
>> rules will be sum of rules in all tables, but new rules only can be
>> added to latest table, so number of rules can be more than size of
>> latest table.
>> - Or user can add more flows after resize() and this may not leave
>> enough room to update old rules to new table, what is expected behavior
>> for this case?
>> - Or if user did not updated rules at all after resize(), after each
>> rule deletion driver won't need to check if old table emptied and needs
>> to be freed?
>> - Or can user call resize() API multiple times, causing driver to
>> maintain multiple tables? How much memory overhead this may bring?
> 
> After "resize, update, complete" sequence table performance must be
> the same as before resize.
> If application skiped updates or resize completion, performance is
> undefined.
>

According below update, it is expected some time will pass until user
updates flows, during this time there may be some performance impact,
does it make sense to mention from it in API documentation?


> Driver must verify that total flows number does not exceed capacity set
> in table resize.
> 

OK, this is more complexity to driver


Is multiple resize() call supported?

In the worst case, think about a case, for each rule application first
increase the size by one and adds that rule. Is this supported and
should there be a limit how many resize() can be done?


>> 'rte_flow_async_update_resized()' API is called per flow, won't this
>> force application to trace which flows are created in new table and
>> which are in old table, so pushing additional work to application.
> 
> Application must trace what flows require update after table resize.
> As the general rule, all flows that were created before table resize
> call has returned must be updated:
> 
> "old" flows |<-----------------resize------------>| "new" flows
>   update                                              keep
>                unknown flow location: update
> 
> ----------------------------TIME-------------------------------------->
> 
> In MLX5 PMD, if update was called with a "new" flow, the call returns
> will success, without changing the flow.
> 

At least this makes life easy for the application, can we document this
behavior as API requirement?

But still in a case there is high amount of flow insertion and deletion
happening in parallel, how application call the update(), it may still
need to keep list of flows to update.

>>
>> Or what will happen if update() fails in the middle of update, should
>> user retry, should PMD restore back the moved rules?
>>
> 
> If flow update call failed, it treated as failure during flow create,
> update or destroy.
> 
>>
>> I understood the logic behind the dividing responsibility to multiple
>> APIs, and it makes sense, but it brings above complexities, and more
>> work to application.
>> Can it be possible to have monolithic API but only resize() part of it
>> is blocking and update() part and later remove table part done
>> asynchronously?
>>
> 
> Table resize and single flow update operations consume approximately the
> same time duration.
>

Ah, thanks. I was missing this bit of information.


> An update of a table with 1_000_000 flows will consume driver for too
> much time.
> During that time application will not be able to create, destroy or
> update existing "old" flows.
> Such operation must be coordinated with application.
> 

Got it. Briefly I was trying to highlight the complexities that multiple
APIs bring, and responsibility pushed to the application,
but above performance impact explains the design decision.

Eventually application needs to get this hit, how application expected
to use these APIs?
First call resize(), after this point how application should handle
updating flows?


Can something like async updating flow work? Like driver returns success
but it sets a thread that in background moves flows in a loop, if it
gets the lock and release it per flow, this lets other thread get the
lock for other flow related operations?

> A driver could provide a batch flows update, but I don’t see how it helps.
> It's ether update one or update all and the latter does not scale.
> 
>>
>> I will also put more comment on the patch based on latest understanding.
>>
>>

Re: [PATCH] ethdev: add template table resize API

[Etelson, Gregory]

[-- Attachment #1: Type: text/plain, Size: 4778 bytes --]

Hello Ferruh,

I work on the patch update with detailed user guide for
the table resize API.

Regards,
Gregory


>>> So, by design, driver will keep the old table when it is resized.
>>> - Can this have a performance impact, like when rules
>>> updated/removed/inserted driver will need to look more tables?
>>> - Or can this cause additional capacity complexity, like total number of
>>> rules will be sum of rules in all tables, but new rules only can be
>>> added to latest table, so number of rules can be more than size of
>>> latest table.
>>> - Or user can add more flows after resize() and this may not leave
>>> enough room to update old rules to new table, what is expected behavior
>>> for this case?
>>> - Or if user did not updated rules at all after resize(), after each
>>> rule deletion driver won't need to check if old table emptied and needs
>>> to be freed?
>>> - Or can user call resize() API multiple times, causing driver to
>>> maintain multiple tables? How much memory overhead this may bring?
>>
>> After "resize, update, complete" sequence table performance must be
>> the same as before resize.
>> If application skiped updates or resize completion, performance is
>> undefined.
>>
>
> According below update, it is expected some time will pass until user
> updates flows, during this time there may be some performance impact,
> does it make sense to mention from it in API documentation?
>
>
>> Driver must verify that total flows number does not exceed capacity set
>> in table resize.
>>
>
> OK, this is more complexity to driver
>
>
> Is multiple resize() call supported?
>
> In the worst case, think about a case, for each rule application first
> increase the size by one and adds that rule. Is this supported and
> should there be a limit how many resize() can be done?
>
>
>>> 'rte_flow_async_update_resized()' API is called per flow, won't this
>>> force application to trace which flows are created in new table and
>>> which are in old table, so pushing additional work to application.
>>
>> Application must trace what flows require update after table resize.
>> As the general rule, all flows that were created before table resize
>> call has returned must be updated:
>>
>> "old" flows |<-----------------resize------------>| "new" flows
>>   update                                              keep
>>                unknown flow location: update
>>
>> ----------------------------TIME-------------------------------------->
>>
>> In MLX5 PMD, if update was called with a "new" flow, the call returns
>> will success, without changing the flow.
>>
>
> At least this makes life easy for the application, can we document this
> behavior as API requirement?
>
> But still in a case there is high amount of flow insertion and deletion
> happening in parallel, how application call the update(), it may still
> need to keep list of flows to update.
>
>>>
>>> Or what will happen if update() fails in the middle of update, should
>>> user retry, should PMD restore back the moved rules?
>>>
>>
>> If flow update call failed, it treated as failure during flow create,
>> update or destroy.
>>
>>>
>>> I understood the logic behind the dividing responsibility to multiple
>>> APIs, and it makes sense, but it brings above complexities, and more
>>> work to application.
>>> Can it be possible to have monolithic API but only resize() part of it
>>> is blocking and update() part and later remove table part done
>>> asynchronously?
>>>
>>
>> Table resize and single flow update operations consume approximately the
>> same time duration.
>>
>
> Ah, thanks. I was missing this bit of information.
>
>
>> An update of a table with 1_000_000 flows will consume driver for too
>> much time.
>> During that time application will not be able to create, destroy or
>> update existing "old" flows.
>> Such operation must be coordinated with application.
>>
>
> Got it. Briefly I was trying to highlight the complexities that multiple
> APIs bring, and responsibility pushed to the application,
> but above performance impact explains the design decision.
>
> Eventually application needs to get this hit, how application expected
> to use these APIs?
> First call resize(), after this point how application should handle
> updating flows?
>
>
> Can something like async updating flow work? Like driver returns success
> but it sets a thread that in background moves flows in a loop, if it
> gets the lock and release it per flow, this lets other thread get the
> lock for other flow related operations?
>
>> A driver could provide a batch flows update, but I don’t see how it helps.
>> It's ether update one or update all and the latter does not scale.
>>
>>>
>>> I will also put more comment on the patch based on latest understanding.
>>>
>>>
>
>

[PATCH v5] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handlers held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update all existing table flow rules by calling
  `rte_flow_async_update_resized()`.
  The table resize procedure does not change application flow handler.
  However, flow object can reference internal PMD resources that are
  obsolete after table resize.
  `rte_flow_async_update_resized()` moves internal flow references
  to the updated table resources.
  The flow update must not interrupt hardware flow operations.

* When all table flow were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
v4: Remove inline.
    Add use case to rte_flow.rst.
v5: update API guide in the rte_flow.rst.	
---
 app/test-pmd/cmdline_flow.c                 |  86 ++++++++++++++-
 app/test-pmd/config.c                       | 102 +++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 +
 doc/guides/howto/rte_flow.rst               | 116 ++++++++++++++++++++
 doc/guides/rel_notes/release_24_03.rst      |   2 +
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  77 +++++++++++++
 lib/ethdev/rte_flow.h                       | 111 +++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   6 +
 12 files changed, 572 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..1a2556d53b 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/howto/rte_flow.rst b/doc/guides/howto/rte_flow.rst
index 27d4f28f77..7cdaadd0ed 100644
--- a/doc/guides/howto/rte_flow.rst
+++ b/doc/guides/howto/rte_flow.rst
@@ -303,3 +303,119 @@ Terminal 1: output log::
   received packet with src ip = 176.80.50.4 sent to queue 3
   received packet with src ip = 176.80.50.5 sent to queue 1
   received packet with src ip = 176.80.50.6 sent to queue 3
+
+Template API resizable table
+----------------------------
+
+Description
+~~~~~~~~~~~
+
+A guide to the resizable template table API.
+
+The resizable template table API enables applications to dynamically adjust
+capacity of template tables without disrupting the existing flows operation.
+The resizable template table API allows applications to optimize the memory
+usage and performance of template tables according to the traffic conditions
+and requirements.
+
+A typical use case for the resizable template table API
+
+  1. Create a resizable table with the initial capacity.
+
+  2. Change the table flows capacity.
+
+  3. Update flows that were created before the table update.
+
+  4. Complete the table resize procedure.
+
+When application begins to resize the table, it enters the resizable state.
+When application finishes resizing the table, it returns to the normal state.
+Only a table in the normal state can be resized. After a table is back to
+the normal state, application can start a new resize.
+Application can add, change or remove flow rules regardless of table state.
+Table performance may worsen in the resizable state. Table performance must
+recover after the table is back to the normal state.
+
+Table resize procedure must not interfere with flows that existed before
+the table size changed.
+Flow handles must remain unchanged during table resize.
+Application must be able to create new flows and modify or delete existing flows
+regardless of the table state.
+
+Application needs to set the `RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE` bit in
+the table attributes when creating a template table that can be resized.
+The current API cannot make an existing table resizable if it was not created
+with the `RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE` bit.
+Resizable template table starts in the normal state.
+
+Application can trigger the table to resize by calling
+the `rte_flow_template_table_resize()` function. The resize process updates
+the PMD table settings and port hardware to fit the new flows capacity.
+The resize process must not affect the current flows functionality.
+The resize process must not change the current flows handles.
+Application can create new flows and modify or delete existing flows
+while the table is resizing, but the table performance might be
+slower than usual.
+
+Flows that existed before table resize are still functional after table resize.
+However, the PMD flow resources that existed before table resize may not be
+fully efficient after table resize. In this case, application can combine
+the old flow resources from before the resize with the new flow resources
+from after the resize.
+Application uses the `rte_flow_async_update_resized()` function call to update
+flow resources. The flow update process does not interfere with or alter
+the existing flow object. It only updates the PMD resources associated with that
+flow.
+The post-resize flow update process may conflict with application flows
+operations, such as creation, removal or update. Therefore, performance-oriented
+applications need to choose the best time to call for post-resize flow update.
+When application selects flows for the post table resize update, it can iterate
+over all existing flows or it can keep track of the flows that need
+to be updated.
+Flows that were created after the `rte_flow_template_table_resize()`
+call finished do not require an update.
+
+To return table to the normal state, use the
+`rte_flow_template_table_resize_complete()`. If PMD does not require post-resize
+flows update and application does not care about PMD resources optimization,
+application can avoid post-resize flows update and move resized table back to
+the normal state right after the `rte_flow_template_table_resize()`.
+Application can resize the table again when it is in the normal state.
+
+Testpmd commands:(wrapped for clarity)::
+
+  # 1. Create resizable template table for 1 flow.
+  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
+                template eth / ipv4 / udp src mask 0xffff / end
+  testpmd> flow actions_template 0 create ingress actions_template_id 7
+                template count  / rss / end
+  testpmd> flow template_table 0 create table_id 101 resizable ingress
+                group 1 priority 0 rules_number 1
+                pattern_template 3 actions_template 7
+
+  # 2. Queue a flow rule.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
+
+  # 3. Resize the template table
+  #    The new table capacity is 32 rules
+  testpmd> flow template_table 0 resize table_resize_id 101
+                table_resize_rules_num 32
+
+  # 4. Queue more flow rules.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
+
+  # 5. Queue the initial flow update.
+  testpmd> flow queue 0 update_resized 0 rule 0
+
+  # 6. Complete the table resize.
+  testpmd> flow template_table 0 resize_complete table 101
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 6f8ad27808..047664a079 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,8 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for template API table resize.**
+
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index f49d1d3767..99987e6518 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2481,3 +2481,80 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..9aedc06516 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,25 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query whether a table can be resized.
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table.
+ *
+ * @return
+ *   True if the table can be resized.
+ */
+__rte_experimental
+bool
+rte_flow_table_resizable(__rte_unused uint16_t port_id,
+			 const struct rte_flow_template_table_attr *tbl_attr);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6773,94 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Change template table flow rules capacity.
+ * PMD implementation must support table change to the new size.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* cannot be resized or resize to *nb_rules*
+ *               is not supported in PMD.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, update flow resources in port.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *rule* cannot be updated.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Following table resize, notify port that all table flows were updated.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) PMD cannot complete table resize.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..5b22aa598a 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,12 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_table_resizable;
+	rte_flow_template_table_resize;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v5] ethdev: add template table resize API

[Thomas Monjalon]

11/02/2024 10:30, Gregory Etelson:
> --- a/doc/guides/howto/rte_flow.rst
> +++ b/doc/guides/howto/rte_flow.rst
> +Template API resizable table
> +----------------------------
> +
> +Description
> +~~~~~~~~~~~
> +
> +A guide to the resizable template table API.

This sentence is useless. Let's keep it as short as we can.

> +
> +The resizable template table API enables applications to dynamically adjust
> +capacity of template tables without disrupting the existing flows operation.
> +The resizable template table API allows applications to optimize the memory
> +usage and performance of template tables according to the traffic conditions
> +and requirements.
> +
> +A typical use case for the resizable template table API

You may add a colon at the end of this sentence.
The below you may use "#" for automatic list numbering.

> +
> +  1. Create a resizable table with the initial capacity.
> +
> +  2. Change the table flows capacity.
> +
> +  3. Update flows that were created before the table update.
> +
> +  4. Complete the table resize procedure.
> +
> +When application begins to resize the table, it enters the resizable state.

I'm not sure to understand what means "enters the resizable state".

> +When application finishes resizing the table, it returns to the normal state.
> +Only a table in the normal state can be resized.

This sentence looks redundant with the following one.

> After a table is back to
> +the normal state, application can start a new resize.
> +Application can add, change or remove flow rules regardless of table state.
> +Table performance may worsen in the resizable state. Table performance must
> +recover after the table is back to the normal state.

This last sentence looks redundant.

> +
> +Table resize procedure must not interfere with flows that existed before
> +the table size changed.

Who the "must" is for? I suppose you are talking about driver requirement?

> +Flow handles must remain unchanged during table resize.

Again, it is for driver behaviour?

> +Application must be able to create new flows and modify or delete existing flows
> +regardless of the table state.

If it is by design we may say "Application is able".

> +
> +Application needs to set the `RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE` bit in

Please use double backquotes for code items.

> +the table attributes when creating a template table that can be resized.
> +The current API cannot make an existing table resizable if it was not created

In the guide, there is no old or current API.

> +with the `RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE` bit.

I think this sentence is redundant.

> +Resizable template table starts in the normal state.

It looks obvious.
Remember a guide is doc a detailed API description.

> +
> +Application can trigger the table to resize by calling
> +the `rte_flow_template_table_resize()` function. The resize process updates
> +the PMD table settings and port hardware to fit the new flows capacity.
> +The resize process must not affect the current flows functionality.
> +The resize process must not change the current flows handles.

Is it redundant with what is above?
Try to be concise.

> +Application can create new flows and modify or delete existing flows
> +while the table is resizing, but the table performance might be
> +slower than usual.

It was already explained before.
I prefer this one.

> +
> +Flows that existed before table resize are still functional after table resize.
> +However, the PMD flow resources that existed before table resize may not be
> +fully efficient after table resize.

What is "not fully efficient" exactly?

> In this case, application can combine
> +the old flow resources from before the resize with the new flow resources
> +from after the resize.
> +Application uses the `rte_flow_async_update_resized()` function call to update
> +flow resources. The flow update process does not interfere with or alter
> +the existing flow object. It only updates the PMD resources associated with that
> +flow.
> +The post-resize flow update process may conflict with application flows
> +operations, such as creation, removal or update. Therefore, performance-oriented
> +applications need to choose the best time to call for post-resize flow update.

Not perfectly clear. What must be done by the app exactly?

> +When application selects flows for the post table resize update, it can iterate
> +over all existing flows or it can keep track of the flows that need
> +to be updated.
> +Flows that were created after the `rte_flow_template_table_resize()`
> +call finished do not require an update.
> +
> +To return table to the normal state, use the
> +`rte_flow_template_table_resize_complete()`. If PMD does not require post-resize
> +flows

How do we know what the PMD requires?

> update and application does not care about PMD resources optimization,
> +application can avoid post-resize flows update and move resized table back to
> +the normal state right after the `rte_flow_template_table_resize()`.
> +Application can resize the table again when it is in the normal state.
> +
> +Testpmd commands:(wrapped for clarity)::

You should replace the first colon with a space.

> +
> +  # 1. Create resizable template table for 1 flow.
> +  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
> +                template eth / ipv4 / udp src mask 0xffff / end
> +  testpmd> flow actions_template 0 create ingress actions_template_id 7
> +                template count  / rss / end
> +  testpmd> flow template_table 0 create table_id 101 resizable ingress
> +                group 1 priority 0 rules_number 1
> +                pattern_template 3 actions_template 7
> +
> +  # 2. Queue a flow rule.
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
> +
> +  # 3. Resize the template table
> +  #    The new table capacity is 32 rules
> +  testpmd> flow template_table 0 resize table_resize_id 101

Why not just "resize table 101" ?

> +                table_resize_rules_num 32
> +
> +  # 4. Queue more flow rules.
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
> +
> +  # 5. Queue the initial flow update.
> +  testpmd> flow queue 0 update_resized 0 rule 0
> +
> +  # 6. Complete the table resize.
> +  testpmd> flow template_table 0 resize_complete table 101
> diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
> index 6f8ad27808..047664a079 100644
> --- a/doc/guides/rel_notes/release_24_03.rst
> +++ b/doc/guides/rel_notes/release_24_03.rst
> @@ -55,6 +55,8 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>  
> +* **Added support for template API table resize.**

Would be better to give a quick description to say it is adding
a new table creation flag and 4 new functions.

[...]
> --- a/lib/ethdev/rte_flow.h
> +++ b/lib/ethdev/rte_flow.h
> +/**
> + * Specialize table for resize.
> + */
> +#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
>  /**@}*/
[...]
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Query whether a table can be resized.
> + *
> + * @param port_id
> + *    Port identifier of Ethernet device.
> + * @param tbl_attr
> + *    Template table.
> + *
> + * @return
> + *   True if the table can be resized.

This is not aligned, probably because of parameters above.

> + */
> +__rte_experimental
> +bool
> +rte_flow_table_resizable(__rte_unused uint16_t port_id,
> +			 const struct rte_flow_template_table_attr *tbl_attr);
> +
[...]
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Change template table flow rules capacity.
> + * PMD implementation must support table change to the new size.

I don't think it is the good place for this PMD comment.

> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table to modify.
> + * @param nb_rules
> + *   New flow rules capacity.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *table* cannot be resized or resize to *nb_rules*
> + *               is not supported in PMD.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize(uint16_t port_id,
> +			       struct rte_flow_template_table *table,
> +			       uint32_t nb_rules,
> +			       struct rte_flow_error *error);
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, update flow resources in port.

You need to give details here about why, when to do it,
and what are the constraints.
It is really better to have all details in doxygen.
The RST is more to explain the global flow of the process.

> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param queue
> + *   Flow queue for async operation.
> + * @param attr
> + *   Async operation attributes.
> + * @param rule
> + *   Flow rule to update.
> + * @param user_data
> + *   The user data that will be returned on async completion event.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *rule* cannot be updated.
> + */
> +__rte_experimental
> +int
> +rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
> +			      const struct rte_flow_op_attr *attr,
> +			      struct rte_flow *rule, void *user_data,
> +			      struct rte_flow_error *error);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, notify port that all table flows were updated.

You need to give details here about when to call it.

> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table that undergoing resize operation.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) PMD cannot complete table resize.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize_complete(uint16_t port_id,
> +					struct rte_flow_template_table *table,
> +					struct rte_flow_error *error);
[...]
> --- a/lib/ethdev/version.map
> +++ b/lib/ethdev/version.map
> +	# added in 24.03
> +	rte_flow_table_resizable;
> +	rte_flow_template_table_resize;
> +	rte_flow_async_update_resized;
> +	rte_flow_template_table_resize_complete;

alphabetical ordering here please

Re: [PATCH v5] ethdev: add template table resize API

[Ferruh Yigit]

On 2/11/2024 9:30 AM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
> RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handlers held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update all existing table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The table resize procedure does not change application flow handler.
>   However, flow object can reference internal PMD resources that are
>   obsolete after table resize.
>   `rte_flow_async_update_resized()` moves internal flow references
>   to the updated table resources.
>   The flow update must not interrupt hardware flow operations.
> 
> * When all table flow were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 
> Testpmd commands:
> 
> * Create resizable template table
> flow template_table <port-id> create table_id <tbl-id> resizable \
>   [transfer|ingress|egres] group <group-id> \
>   rules_number <initial table capacity> \
>   pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
>   actions_template  <at1> [ actions_template <at2> [ ... ]]
> 
> * Resize table:
> flow template_table <tbl-id> resize table_resize_id <tbl-id> \
>   table_resize_rules_num <new table capacity>
> 
> * Queue a flow update:
> flow queue <port-id> update_resized <tbl-id> rule <flow-id>
> 
> * Complete table resize:
> flow template_table <port-id> resize_complete table <tbl-id>
> 
> Signed-off-by: Gregory Etelson <getelson@nvidia.com>
> Acked-by: Ori Kam <orika@nvidia.com>
>

Hi Gregory,

Thanks for the documentation improvement,

As the APIs are documented well and they are experimental, this enables
to properly understand and improve them when another vendor implements
them, I think this is good enough to prooceed with the APIs.

I have two more clarification requests, can you please check them below?

<...>

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, update flow resources in port.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param queue
> + *   Flow queue for async operation.
> + * @param attr
> + *   Async operation attributes.
> + * @param rule
> + *   Flow rule to update.
> + * @param user_data
> + *   The user data that will be returned on async completion event.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *rule* cannot be updated.
> + */
> +__rte_experimental
> +int
> +rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
> +			      const struct rte_flow_op_attr *attr,
> +			      struct rte_flow *rule, void *user_data,
> +			      struct rte_flow_error *error);
> +
>

If one ore more flow failed to update, for any reason, should user retry
the update (in that case we need a retry error maybe) and can user still
call 'rte_flow_template_table_resize_complete()' (for possible next
table resize operation)? Can you please clarify this in your document.


When user calls update() with a flow from new table, API should ignore
it and return success, you mentioned this is what mlx implementation is
doing, what do you think to make this as default API behavior and
document it in above API documentation?


> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Following table resize, notify port that all table flows were updated.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table that undergoing resize operation.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) PMD cannot complete table resize.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize_complete(uint16_t port_id,
> +					struct rte_flow_template_table *table,
> +					struct rte_flow_error *error);
>

If 'rte_flow_template_table_resize_complete()' fails, can application
call another resize()? Is this managed in the application level or
driver level (by returning error to next resize if complete() is not
successful)? Is it something to clarify in your document?

Re: [PATCH v5] ethdev: add template table resize API

[Etelson, Gregory]

Hello Thomas,


>> +
>> +  # 1. Create resizable template table for 1 flow.
>> +  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
>> +                template eth / ipv4 / udp src mask 0xffff / end
>> +  testpmd> flow actions_template 0 create ingress actions_template_id 7
>> +                template count  / rss / end
>> +  testpmd> flow template_table 0 create table_id 101 resizable ingress
>> +                group 1 priority 0 rules_number 1
>> +                pattern_template 3 actions_template 7
>> +
>> +  # 2. Queue a flow rule.
>> +  testpmd> flow queue 0 create 0 template_table 101
>> +                pattern_template 0 actions_template 0 postpone no
>> +                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
>> +
>> +  # 3. Resize the template table
>> +  #    The new table capacity is 32 rules
>> +  testpmd> flow template_table 0 resize table_resize_id 101
>
> Why not just "resize table 101" ?
>

That tokens pattern matched template teble creation.

I'll update the patch.

Re: [PATCH v5] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,


>
> I have two more clarification requests, can you please check them below?
>
>
> If one ore more flow failed to update, for any reason, should user retry
> the update (in that case we need a retry error maybe) and can user still
> call 'rte_flow_template_table_resize_complete()' (for possible next
> table resize operation)? Can you please clarify this in your document.

Please see my comment below.

>
> When user calls update() with a flow from new table, API should ignore
> it and return success, you mentioned this is what mlx implementation is
> doing, what do you think to make this as default API behavior and
> document it in above API documentation?
>
>

I'll update the patch.

>
> If 'rte_flow_template_table_resize_complete()' fails, can application
> call another resize()? Is this managed in the application level or
> driver level (by returning error to next resize if complete() is not
> successful)? Is it something to clarify in your document?
>
>

PMD failures in `rte_flow_async_update_resized()` 
and `rte_flow_template_table_resize_complete()` are not recoverable.

Re: [PATCH v5] ethdev: add template table resize API

[Ferruh Yigit]

On 2/12/2024 3:01 PM, Etelson, Gregory wrote:
> Hello Ferruh,
> 
> 
>>
>> I have two more clarification requests, can you please check them below?
>>
>>
>> If one ore more flow failed to update, for any reason, should user retry
>> the update (in that case we need a retry error maybe) and can user still
>> call 'rte_flow_template_table_resize_complete()' (for possible next
>> table resize operation)? Can you please clarify this in your document.
> 
> Please see my comment below.
> 
>>
>> When user calls update() with a flow from new table, API should ignore
>> it and return success, you mentioned this is what mlx implementation is
>> doing, what do you think to make this as default API behavior and
>> document it in above API documentation?
>>
>>
> 
> I'll update the patch.
> 
>>
>> If 'rte_flow_template_table_resize_complete()' fails, can application
>> call another resize()? Is this managed in the application level or
>> driver level (by returning error to next resize if complete() is not
>> successful)? Is it something to clarify in your document?
>>
>>
> 
> PMD failures in `rte_flow_async_update_resized()` and
> `rte_flow_template_table_resize_complete()` are not recoverable.
> 

OK, just please make sure this is clear in the documentation. Thanks.

[PATCH v6] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
  RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handles held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update table flow rules by calling
  `rte_flow_async_update_resized()`.
  The call reconfigures internal flow resources for the new table
  configuration.
  The flow update must not interrupt hardware flow operations.

* After table flows were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
v4: Remove inline.
    Add use case to rte_flow.rst.
v5: update API guide in the rte_flow.rst.
v6: update docs.
---
 app/test-pmd/cmdline_flow.c                 |  86 +++++++++++++-
 app/test-pmd/config.c                       | 102 +++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 +
 doc/guides/howto/rte_flow.rst               |  94 ++++++++++++++++
 doc/guides/rel_notes/release_24_03.rst      |  12 ++
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  77 +++++++++++++
 lib/ethdev/rte_flow.h                       | 118 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   6 +
 12 files changed, 567 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..1a2556d53b 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/howto/rte_flow.rst b/doc/guides/howto/rte_flow.rst
index 27d4f28f77..96c8a7f692 100644
--- a/doc/guides/howto/rte_flow.rst
+++ b/doc/guides/howto/rte_flow.rst
@@ -303,3 +303,97 @@ Terminal 1: output log::
   received packet with src ip = 176.80.50.4 sent to queue 3
   received packet with src ip = 176.80.50.5 sent to queue 1
   received packet with src ip = 176.80.50.6 sent to queue 3
+
+Template API resizable table
+----------------------------
+
+Description
+~~~~~~~~~~~
+
+The resizable template table API enables applications to dynamically adjust
+capacity of template tables without disrupting the existing flows operation.
+The resizable template table API allows applications to optimize the memory
+usage and performance of template tables according to the traffic conditions
+and requirements.
+
+A typical use case for the resizable template table API:
+
+  1. Create a resizable table with the initial capacity.
+
+  2. Change the table flows capacity.
+
+  3. Update table flows.
+
+  4. Complete the table resize.
+
+A resizable table can be ether in the normal or the resizable state.
+When application begins to resize the table, it state is changed from
+the normal to resizable.
+When application finishes resizing the table, the table state returns to
+the normal state.
+Application can resize a table in the normal state only.
+
+Application needs to set the ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit in
+the table attributes when creating a template table that can be resized.
+A table cannot be converted to resizable if it was created without the
+``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit.
+A resizable table cannot be converted to non-resizable.
+
+Application triggers the table resize by calling
+the ``rte_flow_template_table_resize()`` function. The resize process updates
+the table configuration to fit the new flows capacity.
+Table resize does not change existing flows configuration.
+Application can create new flows and modify or delete existing flows
+while the table is resizing, but the table performance might be
+slower than usual.
+
+Flows that existed before table resize are fully functional after table resize.
+However, application needs to update these flows to match the new table
+configuration.
+Application calls the ``rte_flow_async_update_resized()`` to update flow
+resources for the new table configuration.
+Flows created after table resize match new table configuration.
+Application can track flows that need post-resize update or it can update all
+existing flows. ``rte_flow_async_update_resized()`` returns success for flows
+that were created after table resize.
+
+Application calls ``rte_flow_template_table_resize_complete()`` to return a
+table to the normal state after it completed flows update.
+
+Testpmd commands (wrapped for clarity)::
+
+  # 1. Create resizable template table for 1 flow.
+  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
+                template eth / ipv4 / udp src mask 0xffff / end
+  testpmd> flow actions_template 0 create ingress actions_template_id 7
+                template count  / rss / end
+  testpmd> flow template_table 0 create table_id 101 resizable ingress
+                group 1 priority 0 rules_number 1
+                pattern_template 3 actions_template 7
+
+  # 2. Queue a flow rule.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
+
+  # 3. Resize the template table
+  #    The new table capacity is 32 rules
+  testpmd> flow template_table 0 resize table_resize_id 101
+                table_resize_rules_num 32
+
+  # 4. Queue more flow rules.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
+
+  # 5. Queue the initial flow update.
+  testpmd> flow queue 0 update_resized 0 rule 0
+
+  # 6. Complete the table resize.
+  testpmd> flow template_table 0 resize_complete table 101
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 6f8ad27808..9d5115ed96 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,18 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added  API to change template table flows capacity.**
+
+  * ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE`` table configuration bit.
+    Set when template template must be created with the resizable property.
+  * ``rte_flow_template_table_resizable()``.
+    Query wheather template table can be resized.
+  * ``rte_flow_template_table_resize()``.
+    Reconfigure template table for new flows capacity.
+  * ``rte_flow_async_update_resized()``.
+    Reconfigure flows for the updated table configuration.
+  * ``rte_flow_template_table_resize_complete()``.
+    Complete table resize.
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index f49d1d3767..5077b1a4b0 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2481,3 +2481,80 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+bool
+rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
+				  const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..4ef4951808 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,26 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query whether a table can be resized.
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table.
+ *
+ * @return
+ *    True if the table can be resized.
+ */
+__rte_experimental
+bool
+rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
+				  const struct
+				  rte_flow_template_table_attr *tbl_attr);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6774,100 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update template table for new flow rules capacity.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* is not resizable or
+ *               *table* resize to *nb_rules* is not supported or
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update flow for the new template table configuration after table resize.
+ * Can be called for *rule* created before and after *table* resize.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* was not resized.
+ *               If *rule* cannot be updated after *table* resize,
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Resume normal operational mode after table was resized and
+ * table rules were updated for the new table configuration.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if there are rules that were not updated or
+ *               *table* cannot complete table resize,
+ *               unrecoverable error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..cf56de6ea6 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,12 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resizable;
+	rte_flow_template_table_resize;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v6] ethdev: add template table resize API

[Ferruh Yigit]

On 2/12/2024 6:12 PM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
>   RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handles held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The call reconfigures internal flow resources for the new table
>   configuration.
>   The flow update must not interrupt hardware flow operations.
> 
> * After table flows were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 
> Testpmd commands:
> 
> * Create resizable template table
> flow template_table <port-id> create table_id <tbl-id> resizable \
>   [transfer|ingress|egres] group <group-id> \
>   rules_number <initial table capacity> \
>   pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
>   actions_template  <at1> [ actions_template <at2> [ ... ]]
> 
> * Resize table:
> flow template_table <tbl-id> resize table_resize_id <tbl-id> \
>   table_resize_rules_num <new table capacity>
> 
> * Queue a flow update:
> flow queue <port-id> update_resized <tbl-id> rule <flow-id>
> 
> * Complete table resize:
> flow template_table <port-id> resize_complete table <tbl-id>
> 
> Signed-off-by: Gregory Etelson <getelson@nvidia.com>h
> Acked-by: Ori Kam <orika@nvidia.com>
>

There are some checkpatch warnings [1], but except that looks good to me

Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>


@Thomas was reviewing the patch, I will wait him to proceed.



[1]
[PATCH] ethdev: add template table resize API

WARNING:REPEATED_WORD: Possible repeated word: 'template'
#567: FILE: doc/guides/rel_notes/release_24_03.rst:76:
+    Set when template template must be created with the resizable property.

WARNING:TYPO_SPELLING: 'wheather' may be misspelled - perhaps 'weather'?
#569: FILE: doc/guides/rel_notes/release_24_03.rst:78:
+    Query wheather template table can be resized.
           ^^^^^^^^

ERROR:SPACING: need consistent spacing around '*' (ctx:WxV)
#801: FILE: lib/ethdev/rte_flow.h:5945:
+                         rte_flow_template_table_attr *tbl_attr);
                                                       ^
(better to join to "const struct" line)

Re: [PATCH v6] ethdev: add template table resize API

[Thomas Monjalon]

12/02/2024 19:12, Gregory Etelson:
> --- a/doc/guides/howto/rte_flow.rst
> +++ b/doc/guides/howto/rte_flow.rst
> +Template API resizable table
> +----------------------------
> +
> +Description
> +~~~~~~~~~~~
> +
> +The resizable template table API enables applications to dynamically adjust
> +capacity of template tables without disrupting the existing flows operation.
> +The resizable template table API allows applications to optimize the memory
> +usage and performance of template tables according to the traffic conditions
> +and requirements.
> +
> +A typical use case for the resizable template table API:
> +
> +  1. Create a resizable table with the initial capacity.
> +
> +  2. Change the table flows capacity.
> +
> +  3. Update table flows.

Isn't "flow rules" more appropriate than "flows"?

> +
> +  4. Complete the table resize.
> +
> +A resizable table can be ether in the normal or the resizable state.

s/ether/either/
drop "the"

> +When application begins to resize the table, it state is changed from

s/it/its/

> +the normal to resizable.

to resizable state. (no need from)

> +When application finishes resizing the table, the table state returns to
> +the normal state.

simpler:
until the application finishes the resize procedure.

> +Application can resize a table in the normal state only.
> +
> +Application needs to set the ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit in
> +the table attributes when creating a template table that can be resized.
> +A table cannot be converted to resizable if it was created without the
> +``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit.

Redundant

> +A resizable table cannot be converted to non-resizable.
> +
> +Application triggers the table resize by calling
> +the ``rte_flow_template_table_resize()`` function. The resize process updates
> +the table configuration to fit the new flows capacity.

flows or flow rules?

> +Table resize does not change existing flows configuration.
> +Application can create new flows and modify or delete existing flows

flow rules

> +while the table is resizing, but the table performance might be
> +slower than usual.
> +
> +Flows that existed before table resize are fully functional after table resize.
> +However, application needs to update these flows to match the new table
> +configuration.

This part is not clear.
The rules are functional but needs to be updated. Why?

> +Application calls the ``rte_flow_async_update_resized()`` to update flow
> +resources for the new table configuration.
> +Flows created after table resize match new table configuration.
> +Application can track flows that need post-resize update or it can update all
> +existing flows. ``rte_flow_async_update_resized()`` returns success for flows
> +that were created after table resize.
> +
> +Application calls ``rte_flow_template_table_resize_complete()`` to return a
> +table to the normal state after it completed flows update.
> +
> +Testpmd commands (wrapped for clarity)::
> +
> +  # 1. Create resizable template table for 1 flow.
> +  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
> +                template eth / ipv4 / udp src mask 0xffff / end
> +  testpmd> flow actions_template 0 create ingress actions_template_id 7
> +                template count  / rss / end
> +  testpmd> flow template_table 0 create table_id 101 resizable ingress
> +                group 1 priority 0 rules_number 1
> +                pattern_template 3 actions_template 7
> +
> +  # 2. Queue a flow rule.
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
> +
> +  # 3. Resize the template table
> +  #    The new table capacity is 32 rules
> +  testpmd> flow template_table 0 resize table_resize_id 101
> +                table_resize_rules_num 32
> +
> +  # 4. Queue more flow rules.
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
> +  testpmd> flow queue 0 create 0 template_table 101
> +                pattern_template 0 actions_template 0 postpone no
> +                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
> +
> +  # 5. Queue the initial flow update.
> +  testpmd> flow queue 0 update_resized 0 rule 0
> +
> +  # 6. Complete the table resize.
> +  testpmd> flow template_table 0 resize_complete table 101
> diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
> index 6f8ad27808..9d5115ed96 100644
> --- a/doc/guides/rel_notes/release_24_03.rst
> +++ b/doc/guides/rel_notes/release_24_03.rst
> @@ -55,6 +55,18 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>  
> +* **Added  API to change template table flows capacity.**
> +
> +  * ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE`` table configuration bit.
> +    Set when template template must be created with the resizable property.
> +  * ``rte_flow_template_table_resizable()``.
> +    Query wheather template table can be resized.

whether

> +  * ``rte_flow_template_table_resize()``.
> +    Reconfigure template table for new flows capacity.
> +  * ``rte_flow_async_update_resized()``.
> +    Reconfigure flows for the updated table configuration.
> +  * ``rte_flow_template_table_resize_complete()``.
> +    Complete table resize.
[...]
> --- a/lib/ethdev/rte_flow.h
> +++ b/lib/ethdev/rte_flow.h
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Update flow for the new template table configuration after table resize.
> + * Can be called for *rule* created before and after *table* resize.

I feel we need to explain why it has to be called.

> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param queue
> + *   Flow queue for async operation.
> + * @param attr
> + *   Async operation attributes.
> + * @param rule
> + *   Flow rule to update.
> + * @param user_data
> + *   The user data that will be returned on async completion event.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *table* was not resized.
> + *               If *rule* cannot be updated after *table* resize,
> + *               unrecoverable *table* error.
> + */
> +__rte_experimental
> +int
> +rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
> +			      const struct rte_flow_op_attr *attr,
> +			      struct rte_flow *rule, void *user_data,
> +			      struct rte_flow_error *error);
> +

[PATCH v7] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
  RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handles held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update table flow rules by calling
  `rte_flow_async_update_resized()`.
  The call reconfigures internal flow resources for the new table
  configuration.
  The flow update must not interrupt hardware flow operations.

* After table flows were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
v4: Remove inline.
    Add use case to rte_flow.rst.
v5: Update API guide in the rte_flow.rst.
v6: Update docs.
v7: More API doc updates.
---
 app/test-pmd/cmdline_flow.c                 |  86 +++++++++++++-
 app/test-pmd/config.c                       | 102 +++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 +
 doc/guides/howto/rte_flow.rst               |  88 +++++++++++++++
 doc/guides/rel_notes/release_24_03.rst      |  12 ++
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  77 +++++++++++++
 lib/ethdev/rte_flow.h                       | 119 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   6 +
 12 files changed, 562 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index ce71818705..1a2556d53b 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1323,6 +1329,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1337,6 +1345,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1353,6 +1362,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3344,6 +3354,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3354,13 +3377,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3433,6 +3472,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3525,6 +3570,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10334,6 +10387,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10378,18 +10432,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10411,7 +10472,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10513,7 +10575,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -12569,10 +12632,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -12584,6 +12655,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index cad7537bc6..e589ac614b 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1403,6 +1403,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1693,6 +1706,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2665,6 +2691,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2805,6 +2871,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 9b10a9ea1c..92f21e7776 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/howto/rte_flow.rst b/doc/guides/howto/rte_flow.rst
index 27d4f28f77..e5fc6bf199 100644
--- a/doc/guides/howto/rte_flow.rst
+++ b/doc/guides/howto/rte_flow.rst
@@ -303,3 +303,91 @@ Terminal 1: output log::
   received packet with src ip = 176.80.50.4 sent to queue 3
   received packet with src ip = 176.80.50.5 sent to queue 1
   received packet with src ip = 176.80.50.6 sent to queue 3
+
+Template API resizable table
+----------------------------
+
+Description
+~~~~~~~~~~~
+
+The resizable template table API enables applications to dynamically adjust
+capacity of template tables without disrupting the existing flow rules
+operation. The resizable template table API allows applications to optimize the
+memory usage and performance of template tables according to the traffic
+conditions and requirements.
+
+A typical use case for the resizable template table API:
+
+  #. Create a resizable table with the initial capacity.
+  #. Change the table flow rules capacity.
+  #. Update table flow objects.
+  #. Complete the table resize.
+
+A resizable table can be either in normal or resizable state.
+When application begins to resize the table, its state is changed to resizable.
+The table stays in resizable state until the application finishes resize
+procedure.
+The application can resize a table in the normal state only.
+
+The application needs to set the ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit in
+the table attributes when creating a template table that can be resized,
+and the bit cannot be set or cleared later.
+
+The application triggers the table resize by calling
+the ``rte_flow_template_table_resize()`` function. The resize process updates
+the table configuration to fit the new flow rules capacity.
+Table resize does not change existing flow objects configuration.
+The application can create new flow rules and modify or delete existing flow
+rules while the table is resizing, but the table performance might be
+slower than usual.
+
+Flow rules that existed before table resize are fully functional after
+table resize. However, the application must update flow objects to match
+the new table configuration.
+The application calls ``rte_flow_async_update_resized()`` to update flow object
+for the new table configuration.
+All table flow rules must be updated.
+
+The application calls ``rte_flow_template_table_resize_complete()`` to return a
+table to normal state after it completed flow objects update.
+
+Testpmd commands (wrapped for clarity)::
+
+  # 1. Create resizable template table for 1 flow.
+  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
+                template eth / ipv4 / udp src mask 0xffff / end
+  testpmd> flow actions_template 0 create ingress actions_template_id 7
+                template count  / rss / end
+  testpmd> flow template_table 0 create table_id 101 resizable ingress
+                group 1 priority 0 rules_number 1
+                pattern_template 3 actions_template 7
+
+  # 2. Queue a flow rule.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
+
+  # 3. Resize the template table
+  #    The new table capacity is 32 rules
+  testpmd> flow template_table 0 resize table_resize_id 101
+                table_resize_rules_num 32
+
+  # 4. Queue more flow rules.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
+
+  # 5. Queue flow rules updates.
+  testpmd> flow queue 0 update_resized 0 rule 0
+  testpmd> flow queue 0 update_resized 0 rule 1
+  testpmd> flow queue 0 update_resized 0 rule 2
+  testpmd> flow queue 0 update_resized 0 rule 3
+
+  # 6. Complete the table resize.
+  testpmd> flow template_table 0 resize_complete table 101
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index 39088da303..ec4d317af7 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -55,6 +55,18 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added  API to change template table flows capacity.**
+
+  * ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE`` table configuration bit.
+    Set when template must be created with the resizable property.
+  * ``rte_flow_template_table_resizable()``.
+    Query wheather template table can be resized.
+  * ``rte_flow_template_table_resize()``.
+    Reconfigure template table for new flows capacity.
+  * ``rte_flow_async_update_resized()``.
+    Reconfigure flows for the updated table configuration.
+  * ``rte_flow_template_table_resize_complete()``.
+    Complete table resize.
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 447e28e694..d0d3adf643 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2984,12 +2984,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3010,6 +3019,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index 91f71d868b..1a1f685daa 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -774,3 +774,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index 2cd30d63b7..4b566053b7 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2482,3 +2482,80 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 					hash, error);
 	return flow_err(port_id, ret, error);
 }
+
+bool
+rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
+				  const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 78b6bbb159..9ac8718c86 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5746,6 +5746,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5824,6 +5828,26 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query whether a table can be resized.
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table.
+ *
+ * @return
+ *    True if the table can be resized.
+ */
+__rte_experimental
+bool
+rte_flow_template_table_resizable
+	(__rte_unused uint16_t port_id,
+	 const struct rte_flow_template_table_attr *tbl_attr);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6750,6 +6774,101 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
 			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 			 uint32_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update template table for new flow rules capacity.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* is not resizable or
+ *               *table* resize to *nb_rules* is not supported or
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update flow for the new template table configuration after table resize.
+ * Must be called for each *rule* created before and after *table* resize.
+ * Must be called before rte_flow_template_table_resize_complete().
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* was not resized.
+ *               If *rule* cannot be updated after *table* resize,
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Resume normal operational mode after table was resized and
+ * table rules were updated for the new table configuration.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if there are rules that were not updated or
+ *               *table* cannot complete table resize,
+ *               unrecoverable error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index f35f659503..53d9393575 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -370,6 +370,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_template_table *table,
 		 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
 		 uint32_t *hash, struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+			(struct rte_eth_dev *dev,
+			 struct rte_flow_template_table *table,
+			 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 5c4917c020..cf56de6ea6 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -316,6 +316,12 @@ EXPERIMENTAL {
 	rte_eth_recycle_rx_queue_info_get;
 	rte_flow_group_set_miss_actions;
 	rte_flow_calc_table_hash;
+
+	# added in 24.03
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resizable;
+	rte_flow_template_table_resize;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v7] ethdev: add template table resize API

[Thomas Monjalon]

14/02/2024 15:32, Gregory Etelson:
> +The resizable template table API enables applications to dynamically adjust
> +capacity of template tables without disrupting the existing flow rules
> +operation. The resizable template table API allows applications to optimize the
> +memory usage and performance of template tables according to the traffic
> +conditions and requirements.
> +
> +A typical use case for the resizable template table API:
> +
> +  #. Create a resizable table with the initial capacity.
> +  #. Change the table flow rules capacity.
> +  #. Update table flow objects.
> +  #. Complete the table resize.
> +
> +A resizable table can be either in normal or resizable state.
> +When application begins to resize the table, its state is changed to resizable.
> +The table stays in resizable state until the application finishes resize
> +procedure.
> +The application can resize a table in the normal state only.
> +
> +The application needs to set the ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit in
> +the table attributes when creating a template table that can be resized,
> +and the bit cannot be set or cleared later.
> +
> +The application triggers the table resize by calling
> +the ``rte_flow_template_table_resize()`` function. The resize process updates
> +the table configuration to fit the new flow rules capacity.
> +Table resize does not change existing flow objects configuration.
> +The application can create new flow rules and modify or delete existing flow
> +rules while the table is resizing, but the table performance might be
> +slower than usual.
> +
> +Flow rules that existed before table resize are fully functional after
> +table resize. However, the application must update flow objects to match
> +the new table configuration.
> +The application calls ``rte_flow_async_update_resized()`` to update flow object
> +for the new table configuration.
> +All table flow rules must be updated.
> +
> +The application calls ``rte_flow_template_table_resize_complete()`` to return a
> +table to normal state after it completed flow objects update.

[...]
> + * Update flow for the new template table configuration after table resize.
> + * Must be called for each *rule* created before and after *table* resize.
> + * Must be called before rte_flow_template_table_resize_complete().

That's a clear explanation.

Acked-by: Thomas Monjalon <thomas@monjalon.net>

Re: [PATCH v7] ethdev: add template table resize API

[Ferruh Yigit]

On 2/14/2024 2:32 PM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
>   RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handles held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The call reconfigures internal flow resources for the new table
>   configuration.
>   The flow update must not interrupt hardware flow operations.
> 
> * After table flows were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 
> Testpmd commands:
> 
> * Create resizable template table
> flow template_table <port-id> create table_id <tbl-id> resizable \
>   [transfer|ingress|egres] group <group-id> \
>   rules_number <initial table capacity> \
>   pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
>   actions_template  <at1> [ actions_template <at2> [ ... ]]
> 
> * Resize table:
> flow template_table <tbl-id> resize table_resize_id <tbl-id> \
>   table_resize_rules_num <new table capacity>
> 
> * Queue a flow update:
> flow queue <port-id> update_resized <tbl-id> rule <flow-id>
> 
> * Complete table resize:
> flow template_table <port-id> resize_complete table <tbl-id>
> 
> Signed-off-by: Gregory Etelson <getelson@nvidia.com>
> Acked-by: Ori Kam <orika@nvidia.com>
> Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>
> ---
> v2: Update the patch comment.
>     Add table resize commands to testpmd user guide.
> v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
> v4: Remove inline.
>     Add use case to rte_flow.rst.
> v5: Update API guide in the rte_flow.rst.
> v6: Update docs.
> v7: More API doc updates.
> ---
>  app/test-pmd/cmdline_flow.c                 |  86 +++++++++++++-
>  app/test-pmd/config.c                       | 102 +++++++++++++++++
>  app/test-pmd/testpmd.h                      |   6 +
>  doc/guides/howto/rte_flow.rst               |  88 +++++++++++++++
>  doc/guides/rel_notes/release_24_03.rst      |  12 ++
>  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
>  lib/ethdev/ethdev_trace.h                   |  33 ++++++
>  lib/ethdev/ethdev_trace_points.c            |   9 ++
>  lib/ethdev/rte_flow.c                       |  77 +++++++++++++
>  lib/ethdev/rte_flow.h                       | 119 ++++++++++++++++++++
>  lib/ethdev/rte_flow_driver.h                |  15 +++
>  lib/ethdev/version.map                      |   6 +
>  12 files changed, 562 insertions(+), 6 deletions(-)
> 

Having conflict while applying the patch, can you please rebase it on
latest 'next-net'?

<...>

> diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
> index 39088da303..ec4d317af7 100644
> --- a/doc/guides/rel_notes/release_24_03.rst
> +++ b/doc/guides/rel_notes/release_24_03.rst
> @@ -55,6 +55,18 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>  
> +* **Added  API to change template table flows capacity.**
> +
> +  * ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE`` table configuration bit.
> +    Set when template must be created with the resizable property.
> +  * ``rte_flow_template_table_resizable()``.
> +    Query wheather template table can be resized.
>

s/wheather/whether/

<...>

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Query whether a table can be resized.
> + *
> + * @param port_id
> + *    Port identifier of Ethernet device.
> + * @param tbl_attr
> + *    Template table.
> + *
> + * @return
> + *    True if the table can be resized.
> + */
> +__rte_experimental
> +bool
> +rte_flow_template_table_resizable
> +	(__rte_unused uint16_t port_id,
> +	 const struct rte_flow_template_table_attr *tbl_attr);
> +

Syntax above is odd, why not move parenthesis to first line.

<...>

>  /**
>   * @warning
>   * @b EXPERIMENTAL: this API may change without prior notice.
> @@ -6750,6 +6774,101 @@ rte_flow_calc_table_hash(uint16_t port_id, const struct rte_flow_template_table
>  			 const struct rte_flow_item pattern[], uint8_t pattern_template_index,
>  			 uint32_t *hash, struct rte_flow_error *error);
>  
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Update template table for new flow rules capacity.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table to modify.
> + * @param nb_rules
> + *   New flow rules capacity.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if *table* is not resizable or
> + *               *table* resize to *nb_rules* is not supported or
> + *               unrecoverable *table* error.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize(uint16_t port_id,
> +			       struct rte_flow_template_table *table,
> +			       uint32_t nb_rules,
> +			       struct rte_flow_error *error);
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Update flow for the new template table configuration after table resize.
> + * Must be called for each *rule* created before and after *table* resize.
>

This is different than previous version, I just want to confirm if this
is intentional.

Rules created *before* resize must be updated.
Rules created *after* resize not need to be updated, but if API accepts
them and just returns a quick success, this helps user and even user may
prefer to not keep track of flows as before and after resize, in a good
time user can call update for all flows.

But I am not clear why API is saying all flows (before or after) *must*
be updated?
If user is already keeping record of flows *before* resize, why not let
app to update only those flows?

Is the enforcing update of all flows done intentionally, or is it just
wording error?

<...>

> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Resume normal operational mode after table was resized and
> + * table rules were updated for the new table configuration.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param table
> + *   Template table that undergoing resize operation.
> + * @param error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   - (0) if success.
> + *   - (-ENODEV) if *port_id* invalid.
> + *   - (-ENOTSUP) if underlying device does not support this functionality.
> + *   - (-EINVAL) if there are rules that were not updated or
> + *               *table* cannot complete table resize,
> + *               unrecoverable error.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_resize_complete(uint16_t port_id,
> +					struct rte_flow_template_table *table,
> +					struct rte_flow_error *error);

I think I asked this before but perhaps missed the response,
it is possible that user missed to update all flows and called this API,
it will return -EINVAL, in this case it is not clear for user if there
is a unrecoverable error or just need to update more flows.

What do you think to send a different error for the case that there are
flows not updated, so user can action on this information.

Re: [PATCH v7] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,

>
> Having conflict while applying the patch, can you please rebase it on
> latest 'next-net'?

Will rebase and update the patch.

>
>> +    Query wheather template table can be resized.
>>
>
> s/wheather/whether/
>
> <...>

fix in updated patch

>
>> +__rte_experimental
>> +bool
>> +rte_flow_template_table_resizable
>> +     (__rte_unused uint16_t port_id,
>> +      const struct rte_flow_template_table_attr *tbl_attr);
>> +
>
> Syntax above is odd, why not move parenthesis to first line.

Agree that's odd style.
DPDK prefers that way.
Please check rte_flow_driver.h::rte_flow.ops.

>
> This is different than previous version, I just want to confirm if this
> is intentional.

That API version was discussed with Thomas.
The alternative was to introduce additional function call to query
whether a flow rule needs conversion after table resize.
As the result, the application still needs to iterate on all
table flow rules.

>
> Rules created *before* resize must be updated.

That part remains. The application must update
flow rules created before resize.

> Rules created *after* resize not need to be updated, but if API accepts
> them and just returns a quick success,

Calling update for new rules will return success without flow update.

>this helps user and even user may
> prefer to not keep track of flows as before and after resize,

The application now does not need to differentiate table flow rules.
All table flow rules must be updated after table resize.

>
>> +/**
>> + * @warning
>> + * @b EXPERIMENTAL: this API may change without prior notice.
>> + *
>> + * Resume normal operational mode after table was resized and
>> + * table rules were updated for the new table configuration.
>> + *
>> + * @param port_id
>> + *   Port identifier of Ethernet device.
>> + * @param table
>> + *   Template table that undergoing resize operation.
>> + * @param error
>> + *   Perform verbose error reporting if not NULL.
>> + *   PMDs initialize this structure in case of error only.
>> + *
>> + * @return
>> + *   - (0) if success.
>> + *   - (-ENODEV) if *port_id* invalid.
>> + *   - (-ENOTSUP) if underlying device does not support this functionality.
>> + *   - (-EINVAL) if there are rules that were not updated or
>> + *               *table* cannot complete table resize,
>> + *               unrecoverable error.
>> + */
>> +__rte_experimental
>> +int
>> +rte_flow_template_table_resize_complete(uint16_t port_id,
>> +                                     struct rte_flow_template_table *table,
>> +                                     struct rte_flow_error *error);
>
> I think I asked this before but perhaps missed the response,
> it is possible that user missed to update all flows and called this API,
> it will return -EINVAL, in this case it is not clear for user if there
> is a unrecoverable error or just need to update more flows.
>
> What do you think to send a different error for the case that there are
> flows not updated, so user can action on this information.
>
>

A different error is good.
What about EBUSY ?

Re: [PATCH v7] ethdev: add template table resize API

[Ferruh Yigit]

On 2/14/2024 5:07 PM, Etelson, Gregory wrote:
> Hello Ferruh,
> 
>>
>> Having conflict while applying the patch, can you please rebase it on
>> latest 'next-net'?
> 
> Will rebase and update the patch.
> 
>>
>>> +    Query wheather template table can be resized.
>>>
>>
>> s/wheather/whether/
>>
>> <...>
> 
> fix in updated patch
> 
>>
>>> +__rte_experimental
>>> +bool
>>> +rte_flow_template_table_resizable
>>> +     (__rte_unused uint16_t port_id,
>>> +      const struct rte_flow_template_table_attr *tbl_attr);
>>> +
>>
>> Syntax above is odd, why not move parenthesis to first line.
> 
> Agree that's odd style.
> DPDK prefers that way.
> Please check rte_flow_driver.h::rte_flow.ops.
> 

True that is also odd but at least it is function pointers in a struct,
but this is regular function deceleration, why not:
```
bool
rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
         const struct rte_flow_template_table_attr *tbl_attr);
```

>>
>> This is different than previous version, I just want to confirm if this
>> is intentional.
> 
> That API version was discussed with Thomas.
> The alternative was to introduce additional function call to query
> whether a flow rule needs conversion after table resize.
> As the result, the application still needs to iterate on all
> table flow rules.
> 

Either it or application needs to manage it. Agree that application can
call update() on all flows is easier for app, that is why I asked for
this clarification.

But still it is different than API definition making it mandatory to
call for new flows.

What about something like, although it is a little longer:
```
Update flow for the new template table configuration after table resize.

Should be called for rules created before table resize. If called for
rules crated after table resize, API should return success, so
application is free to call this API for all flows.
```


>>
>> Rules created *before* resize must be updated.
> 
> That part remains. The application must update
> flow rules created before resize.
> 
>> Rules created *after* resize not need to be updated, but if API accepts
>> them and just returns a quick success,
> 
> Calling update for new rules will return success without flow update.
> 
>> this helps user and even user may
>> prefer to not keep track of flows as before and after resize,
> 
> The application now does not need to differentiate table flow rules.
> All table flow rules must be updated after table resize.
> 
>>
>>> +/**
>>> + * @warning
>>> + * @b EXPERIMENTAL: this API may change without prior notice.
>>> + *
>>> + * Resume normal operational mode after table was resized and
>>> + * table rules were updated for the new table configuration.
>>> + *
>>> + * @param port_id
>>> + *   Port identifier of Ethernet device.
>>> + * @param table
>>> + *   Template table that undergoing resize operation.
>>> + * @param error
>>> + *   Perform verbose error reporting if not NULL.
>>> + *   PMDs initialize this structure in case of error only.
>>> + *
>>> + * @return
>>> + *   - (0) if success.
>>> + *   - (-ENODEV) if *port_id* invalid.
>>> + *   - (-ENOTSUP) if underlying device does not support this
>>> functionality.
>>> + *   - (-EINVAL) if there are rules that were not updated or
>>> + *               *table* cannot complete table resize,
>>> + *               unrecoverable error.
>>> + */
>>> +__rte_experimental
>>> +int
>>> +rte_flow_template_table_resize_complete(uint16_t port_id,
>>> +                                     struct rte_flow_template_table
>>> *table,
>>> +                                     struct rte_flow_error *error);
>>
>> I think I asked this before but perhaps missed the response,
>> it is possible that user missed to update all flows and called this API,
>> it will return -EINVAL, in this case it is not clear for user if there
>> is a unrecoverable error or just need to update more flows.
>>
>> What do you think to send a different error for the case that there are
>> flows not updated, so user can action on this information.
>>
>>
> 
> A different error is good.
> What about EBUSY ?
>

I don't know, can be EBUSY or EAGAIN.
Or we can overload EEXIST as there are existing not updated flows.

Re: [PATCH v7] ethdev: add template table resize API

[Etelson, Gregory]

Hello Ferruh,

<...>
>>>
>>> Syntax above is odd, why not move parenthesis to first line.
>>
>> Agree that's odd style.
>> DPDK prefers that way.
>> Please check rte_flow_driver.h::rte_flow.ops.
>>
>
> True that is also odd but at least it is function pointers in a struct,
> but this is regular function deceleration, why not:
> ```
> bool
> rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
>         const struct rte_flow_template_table_attr *tbl_attr);
> ```
>

will change in the next patch update.

<...>
> Either it or application needs to manage it. Agree that application can
> call update() on all flows is easier for app, that is why I asked for
> this clarification.
>
> But still it is different than API definition making it mandatory to
> call for new flows.
>
> What about something like, although it is a little longer:
> ```
> Update flow for the new template table configuration after table resize.
>
> Should be called for rules created before table resize. If called for
> rules crated after table resize, API should return success, so
> application is free to call this API for all flows.
> ```
>
>

That's very good.
I'll use it in the next patch update.

<...>
>>
>> A different error is good.
>> What about EBUSY ?
>>
>
> I don't know, can be EBUSY or EAGAIN.
> Or we can overload EEXIST as there are existing not updated flows.
>

According to errno(3):
EAGAIN        Resource  temporarily  unavailable
EBUSY         Device or resource busy
EEXIST        File exists

UBUSY fits the context.

[PATCH v8] ethdev: add template table resize API

[Gregory Etelson]

Template table creation API sets table flows capacity.
If application needs more flows then the table was designed for,
the following procedures must be completed:
1. Create a new template table with larger flows capacity.
2. Re-create existing flows in the new table and delete flows from
   the original table.
3. Destroy original table.

Application cannot always execute that procedure:
* Port may not have sufficient resources to allocate a new table
  while maintaining original table.
* Application may not have existing flows "recipes" to re-create
  flows in a new table.

The patch defines a new API that allows application to resize
existing template table:

* Resizable template table must be created with the
  RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.

* Application resizes existing table with the
  `rte_flow_template_table_resize()` function call.
  The table resize procedure updates the table maximal flow number
  only. Other table attributes are not affected by the table resize.
  ** The table resize procedure must not interrupt
     existing table flows operations in hardware.
  ** The table resize procedure must not alter flow handles held by
     application.

* After `rte_flow_template_table_resize()` returned, application must
  update table flow rules by calling
  `rte_flow_async_update_resized()`.
  The call reconfigures internal flow resources for the new table
  configuration.
  The flow update must not interrupt hardware flow operations.

* After table flows were updated, application must call
  `rte_flow_template_table_resize_complete()`.
  The function releases PMD resources related to the original
  table.
  Application can start new table resize after
  `rte_flow_template_table_resize_complete()` returned.

Testpmd commands:

* Create resizable template table
flow template_table <port-id> create table_id <tbl-id> resizable \
  [transfer|ingress|egres] group <group-id> \
  rules_number <initial table capacity> \
  pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
  actions_template  <at1> [ actions_template <at2> [ ... ]]

* Resize table:
flow template_table <tbl-id> resize table_resize_id <tbl-id> \
  table_resize_rules_num <new table capacity>

* Queue a flow update:
flow queue <port-id> update_resized <tbl-id> rule <flow-id>

* Complete table resize:
flow template_table <port-id> resize_complete table <tbl-id>

Signed-off-by: Gregory Etelson <getelson@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>
Acked-by: Thomas Monjalon <thomas@monjalon.net>
---
v2: Update the patch comment.
    Add table resize commands to testpmd user guide.
v3: Rename RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE macro.
v4: Remove inline.
    Add use case to rte_flow.rst.
v5: Update API guide in the rte_flow.rst.
v6: Update docs.
v7: More API doc updates.
v8: resize_complete returns EBUSY.
---
 app/test-pmd/cmdline_flow.c                 |  86 +++++++++++++-
 app/test-pmd/config.c                       | 102 +++++++++++++++++
 app/test-pmd/testpmd.h                      |   6 +
 doc/guides/howto/rte_flow.rst               |  91 +++++++++++++++
 doc/guides/rel_notes/release_24_03.rst      |  12 ++
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++-
 lib/ethdev/ethdev_trace.h                   |  33 ++++++
 lib/ethdev/ethdev_trace_points.c            |   9 ++
 lib/ethdev/rte_flow.c                       |  77 +++++++++++++
 lib/ethdev/rte_flow.h                       | 118 ++++++++++++++++++++
 lib/ethdev/rte_flow_driver.h                |  15 +++
 lib/ethdev/version.map                      |   4 +
 12 files changed, 562 insertions(+), 6 deletions(-)

diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
index 7370d5f224..8e5eb41d71 100644
--- a/app/test-pmd/cmdline_flow.c
+++ b/app/test-pmd/cmdline_flow.c
@@ -134,6 +134,7 @@ enum index {
 	/* Queue arguments. */
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -191,8 +192,12 @@ enum index {
 	/* Table arguments. */
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	TABLE_CREATE_ID,
 	TABLE_DESTROY_ID,
+	TABLE_RESIZE_ID,
+	TABLE_RESIZE_RULES_NUMBER,
 	TABLE_INSERTION_TYPE,
 	TABLE_INSERTION_TYPE_NAME,
 	TABLE_HASH_FUNC,
@@ -204,6 +209,7 @@ enum index {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1361,6 +1367,8 @@ static const enum index next_group_attr[] = {
 static const enum index next_table_subcmd[] = {
 	TABLE_CREATE,
 	TABLE_DESTROY,
+	TABLE_RESIZE,
+	TABLE_RESIZE_COMPLETE,
 	ZERO,
 };
 
@@ -1375,6 +1383,7 @@ static const enum index next_table_attr[] = {
 	TABLE_TRANSFER,
 	TABLE_TRANSFER_WIRE_ORIG,
 	TABLE_TRANSFER_VPORT_ORIG,
+	TABLE_RESIZABLE,
 	TABLE_RULES_NUMBER,
 	TABLE_PATTERN_TEMPLATE,
 	TABLE_ACTIONS_TEMPLATE,
@@ -1391,6 +1400,7 @@ static const enum index next_table_destroy_attr[] = {
 static const enum index next_queue_subcmd[] = {
 	QUEUE_CREATE,
 	QUEUE_DESTROY,
+	QUEUE_FLOW_UPDATE_RESIZED,
 	QUEUE_UPDATE,
 	QUEUE_AGED,
 	QUEUE_INDIRECT_ACTION,
@@ -3460,6 +3470,19 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE] = {
+		.name = "resize",
+		.help = "resize template table",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_ID)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_COMPLETE] = {
+		.name = "resize_complete",
+		.help = "complete table resize",
+		.next = NEXT(NEXT_ENTRY(TABLE_DESTROY_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, port)),
+		.call = parse_table_destroy,
+	},
 	/* Table  arguments. */
 	[TABLE_CREATE_ID] = {
 		.name = "table_id",
@@ -3470,13 +3493,29 @@ static const struct token token_list[] = {
 	},
 	[TABLE_DESTROY_ID] = {
 		.name = "table",
-		.help = "specify table id to destroy",
+		.help = "table id",
 		.next = NEXT(next_table_destroy_attr,
 			     NEXT_ENTRY(COMMON_TABLE_ID)),
 		.args = ARGS(ARGS_ENTRY_PTR(struct buffer,
 					    args.table_destroy.table_id)),
 		.call = parse_table_destroy,
 	},
+	[TABLE_RESIZE_ID] = {
+		.name = "table_resize_id",
+		.help = "table resize id",
+		.next = NEXT(NEXT_ENTRY(TABLE_RESIZE_RULES_NUMBER),
+			     NEXT_ENTRY(COMMON_TABLE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, args.table.id)),
+		.call = parse_table
+	},
+	[TABLE_RESIZE_RULES_NUMBER] = {
+		.name = "table_resize_rules_num",
+		.help = "table resize rules number",
+		.next = NEXT(NEXT_ENTRY(END), NEXT_ENTRY(COMMON_UNSIGNED)),
+		.args = ARGS(ARGS_ENTRY(struct buffer,
+					args.table.attr.nb_flows)),
+		.call = parse_table
+	},
 	[TABLE_INSERTION_TYPE] = {
 		.name = "insertion_type",
 		.help = "specify insertion type",
@@ -3549,6 +3588,12 @@ static const struct token token_list[] = {
 		.next = NEXT(next_table_attr),
 		.call = parse_table,
 	},
+	[TABLE_RESIZABLE] = {
+		.name = "resizable",
+		.help = "set resizable attribute",
+		.next = NEXT(next_table_attr),
+		.call = parse_table,
+	},
 	[TABLE_RULES_NUMBER] = {
 		.name = "rules_number",
 		.help = "number of rules in table",
@@ -3641,6 +3686,14 @@ static const struct token token_list[] = {
 		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
 		.call = parse_qo_destroy,
 	},
+	[QUEUE_FLOW_UPDATE_RESIZED] = {
+		.name = "update_resized",
+		.help = "update a flow after table resize",
+		.next = NEXT(NEXT_ENTRY(QUEUE_DESTROY_ID),
+			     NEXT_ENTRY(COMMON_QUEUE_ID)),
+		.args = ARGS(ARGS_ENTRY(struct buffer, queue)),
+		.call = parse_qo_destroy,
+	},
 	[QUEUE_UPDATE] = {
 		.name = "update",
 		.help = "update a flow rule",
@@ -10780,6 +10833,7 @@ parse_table(struct context *ctx, const struct token *token,
 	}
 	switch (ctx->curr) {
 	case TABLE_CREATE:
+	case TABLE_RESIZE:
 		out->command = ctx->curr;
 		ctx->objdata = 0;
 		ctx->object = out;
@@ -10824,18 +10878,25 @@ parse_table(struct context *ctx, const struct token *token,
 	case TABLE_TRANSFER_WIRE_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_WIRE_ORIG;
 		return len;
 	case TABLE_TRANSFER_VPORT_ORIG:
 		if (!out->args.table.attr.flow_attr.transfer)
 			return -1;
-		out->args.table.attr.specialize = RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		out->args.table.attr.specialize |= RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG;
+		return len;
+	case TABLE_RESIZABLE:
+		out->args.table.attr.specialize |=
+			RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE;
 		return len;
 	case TABLE_RULES_NUMBER:
 		ctx->objdata = 0;
 		ctx->object = out;
 		ctx->objmask = NULL;
 		return len;
+	case TABLE_RESIZE_ID:
+	case TABLE_RESIZE_RULES_NUMBER:
+		return len;
 	default:
 		return -1;
 	}
@@ -10857,7 +10918,8 @@ parse_table_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == TABLE) {
-		if (ctx->curr != TABLE_DESTROY)
+		if (ctx->curr != TABLE_DESTROY &&
+		    ctx->curr != TABLE_RESIZE_COMPLETE)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -10959,7 +11021,8 @@ parse_qo_destroy(struct context *ctx, const struct token *token,
 	if (!out)
 		return len;
 	if (!out->command || out->command == QUEUE) {
-		if (ctx->curr != QUEUE_DESTROY)
+		if (ctx->curr != QUEUE_DESTROY &&
+		    ctx->curr != QUEUE_FLOW_UPDATE_RESIZED)
 			return -1;
 		if (sizeof(*out) > size)
 			return -1;
@@ -13057,10 +13120,18 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.table_destroy.table_id_n,
 					in->args.table_destroy.table_id);
 		break;
+	case TABLE_RESIZE_COMPLETE:
+		port_flow_template_table_resize_complete
+			(in->port, in->args.table_destroy.table_id[0]);
+		break;
 	case GROUP_SET_MISS_ACTIONS:
 		port_queue_group_set_miss_actions(in->port, &in->args.vc.attr,
 						  in->args.vc.actions);
 		break;
+	case TABLE_RESIZE:
+		port_flow_template_table_resize(in->port, in->args.table.id,
+						in->args.table.attr.nb_flows);
+		break;
 	case QUEUE_CREATE:
 		port_queue_flow_create(in->port, in->queue, in->postpone,
 			in->args.vc.table_id, in->args.vc.rule_id,
@@ -13072,6 +13143,11 @@ cmd_flow_parsed(const struct buffer *in)
 					in->args.destroy.rule_n,
 					in->args.destroy.rule);
 		break;
+	case QUEUE_FLOW_UPDATE_RESIZED:
+		port_queue_flow_update_resized(in->port, in->queue,
+					       in->postpone,
+					       in->args.destroy.rule[0]);
+		break;
 	case QUEUE_UPDATE:
 		port_queue_flow_update(in->port, in->queue, in->postpone,
 				in->args.vc.rule_id, in->args.vc.act_templ_id,
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index 9732cb2e0c..08d7d0e2f8 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1404,6 +1404,19 @@ port_flow_new(const struct rte_flow_attr *attr,
 	return NULL;
 }
 
+static struct port_flow *
+port_flow_locate(struct port_flow *flows_list, uint32_t flow_id)
+{
+	struct port_flow *pf = flows_list;
+
+	while (pf) {
+		if (pf->id == flow_id)
+			break;
+		pf = pf->next;
+	}
+	return pf;
+}
+
 /** Print a message out of a flow error. */
 static int
 port_flow_complain(struct rte_flow_error *error)
@@ -1694,6 +1707,19 @@ table_alloc(uint32_t id, struct port_table **table,
 	return 0;
 }
 
+static struct port_table *
+port_table_locate(struct port_table *tables_list, uint32_t table_id)
+{
+	struct port_table *pt = tables_list;
+
+	while (pt) {
+		if (pt->id == table_id)
+			break;
+		pt = pt->next;
+	}
+	return pt;
+}
+
 /** Get info about flow management resources. */
 int
 port_flow_get_info(portid_t port_id)
@@ -2666,6 +2692,46 @@ port_flow_template_table_destroy(portid_t port_id,
 	return ret;
 }
 
+int
+port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize_complete(port_id,
+						      pt->table, &error);
+	return !ret ? 0 : port_flow_complain(&error);
+}
+
+int
+port_flow_template_table_resize(portid_t port_id,
+				uint32_t table_id, uint32_t flows_num)
+{
+	struct rte_port *port;
+	struct port_table *pt;
+	struct rte_flow_error error = { 0, };
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -EINVAL;
+	port = &ports[port_id];
+	pt = port_table_locate(port->table_list, table_id);
+	if (!pt)
+		return -EINVAL;
+	ret = rte_flow_template_table_resize(port_id, pt->table, flows_num, &error);
+	if (ret)
+		return port_flow_complain(&error);
+	return 0;
+}
+
 /** Flush table */
 int
 port_flow_template_table_flush(portid_t port_id)
@@ -2806,6 +2872,42 @@ port_queue_flow_create(portid_t port_id, queueid_t queue_id,
 	return 0;
 }
 
+int
+port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+			       bool postpone, uint32_t flow_id)
+{
+	const struct rte_flow_op_attr op_attr = { .postpone = postpone };
+	struct rte_flow_error error = { 0, };
+	struct port_flow *pf;
+	struct rte_port *port;
+	struct queue_job *job;
+	int ret;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
+	    port_id == (portid_t)RTE_PORT_ALL)
+		return -EINVAL;
+	port = &ports[port_id];
+	if (queue_id >= port->queue_nb) {
+		printf("Queue #%u is invalid\n", queue_id);
+		return -EINVAL;
+	}
+	pf = port_flow_locate(port->flow_list, flow_id);
+	if (!pf)
+		return -EINVAL;
+	job = calloc(1, sizeof(*job));
+	if (!job)
+		return -ENOMEM;
+	job->type = QUEUE_JOB_TYPE_FLOW_TRANSFER;
+	job->pf = pf;
+	ret = rte_flow_async_update_resized(port_id, queue_id, &op_attr,
+					    pf->flow, job, &error);
+	if (ret) {
+		free(job);
+		return port_flow_complain(&error);
+	}
+	return 0;
+}
+
 /** Enqueue number of destroy flow rules operations. */
 int
 port_queue_flow_destroy(portid_t port_id, queueid_t queue_id,
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index e18546c46e..467ef3a8ab 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -110,6 +110,7 @@ enum {
 enum {
 	QUEUE_JOB_TYPE_FLOW_CREATE,
 	QUEUE_JOB_TYPE_FLOW_DESTROY,
+	QUEUE_JOB_TYPE_FLOW_TRANSFER,
 	QUEUE_JOB_TYPE_FLOW_UPDATE,
 	QUEUE_JOB_TYPE_ACTION_CREATE,
 	QUEUE_JOB_TYPE_ACTION_DESTROY,
@@ -981,7 +982,12 @@ int port_flow_template_table_create(portid_t port_id, uint32_t id,
 		   uint32_t nb_actions_templates, uint32_t *actions_templates);
 int port_flow_template_table_destroy(portid_t port_id,
 			    uint32_t n, const uint32_t *table);
+int port_queue_flow_update_resized(portid_t port_id, queueid_t queue_id,
+				   bool postpone, uint32_t flow_id);
 int port_flow_template_table_flush(portid_t port_id);
+int port_flow_template_table_resize_complete(portid_t port_id, uint32_t table_id);
+int port_flow_template_table_resize(portid_t port_id,
+				    uint32_t table_id, uint32_t flows_num);
 int port_queue_group_set_miss_actions(portid_t port_id, const struct rte_flow_attr *attr,
 				      const struct rte_flow_action *actions);
 int port_queue_flow_create(portid_t port_id, queueid_t queue_id,
diff --git a/doc/guides/howto/rte_flow.rst b/doc/guides/howto/rte_flow.rst
index 27d4f28f77..2b84728054 100644
--- a/doc/guides/howto/rte_flow.rst
+++ b/doc/guides/howto/rte_flow.rst
@@ -303,3 +303,94 @@ Terminal 1: output log::
   received packet with src ip = 176.80.50.4 sent to queue 3
   received packet with src ip = 176.80.50.5 sent to queue 1
   received packet with src ip = 176.80.50.6 sent to queue 3
+
+Template API resizable table
+----------------------------
+
+Description
+~~~~~~~~~~~
+
+The resizable template table API enables applications to dynamically adjust
+capacity of template tables without disrupting the existing flow rules
+operation. The resizable template table API allows applications to optimize the
+memory usage and performance of template tables according to the traffic
+conditions and requirements.
+
+A typical use case for the resizable template table API:
+
+  #. Create a resizable table with the initial capacity.
+  #. Change the table flow rules capacity.
+  #. Update table flow objects.
+  #. Complete the table resize.
+
+A resizable table can be either in normal or resizable state.
+When application begins to resize the table, its state is changed to resizable.
+The table stays in resizable state until the application finishes resize
+procedure.
+The application can resize a table in the normal state only.
+
+The application needs to set the ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE`` bit in
+the table attributes when creating a template table that can be resized,
+and the bit cannot be set or cleared later.
+
+The application triggers the table resize by calling
+the ``rte_flow_template_table_resize()`` function. The resize process updates
+the table configuration to fit the new flow rules capacity.
+Table resize does not change existing flow objects configuration.
+The application can create new flow rules and modify or delete existing flow
+rules while the table is resizing, but the table performance might be
+slower than usual.
+
+Flow rules that existed before table resize are fully functional after
+table resize. However, the application must update flow objects to match
+the new table configuration.
+The application calls ``rte_flow_async_update_resized()`` to update flow object
+for the new table configuration.
+Should be called for flow rules created before table resize. If called for
+flow rules crated after table resize, the call should return success.
+The application is free to call this API for all table flow rules.
+
+The application calls ``rte_flow_template_table_resize_complete()`` to return a
+table to normal state after it completed flow objects update.
+
+Testpmd commands (wrapped for clarity)::
+
+  # 1. Create resizable template table for 1 flow.
+  testpmd> flow pattern_template 0 create ingress pattern_template_id 3
+                template eth / ipv4 / udp src mask 0xffff / end
+  testpmd> flow actions_template 0 create ingress actions_template_id 7
+                template count  / rss / end
+  testpmd> flow template_table 0 create table_id 101 resizable ingress
+                group 1 priority 0 rules_number 1
+                pattern_template 3 actions_template 7
+
+  # 2. Queue a flow rule.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 1 / end actions count / rss / end
+
+  # 3. Resize the template table
+  #    The new table capacity is 32 rules
+  testpmd> flow template_table 0 resize table_resize_id 101
+                table_resize_rules_num 32
+
+  # 4. Queue more flow rules.
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 2 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 3 / end actions count / rss / end
+  testpmd> flow queue 0 create 0 template_table 101
+                pattern_template 0 actions_template 0 postpone no
+                pattern eth / ipv4 / udp src spec 4 / end actions count / rss / end
+
+  # 5. Queue flow rules updates.
+  # Rule 0 was created before table resize - must be updated.
+  testpmd> flow queue 0 update_resized 0 rule 0
+  # Rule 1 was created after table resize - flow pull returns success.
+  testpmd> flow queue 0 update_resized 0 rule 1
+
+
+  # 6. Complete the table resize.
+  testpmd> flow template_table 0 resize_complete table 101
diff --git a/doc/guides/rel_notes/release_24_03.rst b/doc/guides/rel_notes/release_24_03.rst
index bd9b4b3b76..b2c9dcaefc 100644
--- a/doc/guides/rel_notes/release_24_03.rst
+++ b/doc/guides/rel_notes/release_24_03.rst
@@ -89,6 +89,18 @@ New Features
 
   * Added support for comparing result between packet fields or value.
 
+* **Added  API to change template table flows capacity.**
+
+  * ``RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE`` table configuration bit.
+    Set when template must be created with the resizable property.
+  * ``rte_flow_template_table_resizable()``.
+    Query whether template table can be resized.
+  * ``rte_flow_template_table_resize()``.
+    Reconfigure template table for new flows capacity.
+  * ``rte_flow_async_update_resized()``.
+    Reconfigure flows for the updated table configuration.
+  * ``rte_flow_template_table_resize_complete()``.
+    Complete table resize.
 
 Removed Items
 -------------
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 7a42f21ee6..e972b6dc8b 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -2995,12 +2995,21 @@ following sections.
 - Create a table::
 
    flow table {port_id} create
-       [table_id {id}]
+       [table_id {id}] [resizable]
        [group {group_id}] [priority {level}] [ingress] [egress] [transfer]
        rules_number {number}
        pattern_template {pattern_template_id}
        actions_template {actions_template_id}
 
+- Resize a table::
+
+   flow template_table {port_id} resize
+       table_resize_id {id} table_resize_rules_num {number}
+
+- Complete table resize::
+
+   flow template_table {port_id} resize_complete table {table_id}
+
 - Destroy a table::
 
    flow table {port_id} destroy table {id} [...]
@@ -3021,6 +3030,10 @@ following sections.
        pattern {item} [/ {item} [...]] / end
        actions {action} [/ {action} [...]] / end
 
+- Enqueue flow update following table resize::
+
+   flow queue {port_id} update_resized {table_id} rule {rule_id}
+
 - Enqueue destruction of specific flow rules::
 
    flow queue {port_id} destroy {queue_id}
diff --git a/lib/ethdev/ethdev_trace.h b/lib/ethdev/ethdev_trace.h
index 1b1ae0cfe8..cd3327a619 100644
--- a/lib/ethdev/ethdev_trace.h
+++ b/lib/ethdev/ethdev_trace.h
@@ -2572,6 +2572,39 @@ RTE_TRACE_POINT_FP(
 	rte_trace_point_emit_ptr(user_data);
 	rte_trace_point_emit_int(ret);
 )
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_template_table_resize,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table,
+			     uint32_t nb_rules, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_u32(nb_rules);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_async_update_resized,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id, uint32_t queue,
+			     const struct rte_flow_op_attr *attr,
+			     struct rte_flow *rule, void *user_data, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_u32(queue);
+	rte_trace_point_emit_ptr(attr);
+	rte_trace_point_emit_ptr(rule);
+	rte_trace_point_emit_ptr(user_data);
+	rte_trace_point_emit_int(ret);
+)
+
+RTE_TRACE_POINT_FP(
+	rte_flow_trace_table_resize_complete,
+	RTE_TRACE_POINT_ARGS(uint16_t port_id,
+			     struct rte_flow_template_table *table, int ret),
+	rte_trace_point_emit_u16(port_id);
+	rte_trace_point_emit_ptr(table);
+	rte_trace_point_emit_int(ret);
+)
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/ethdev_trace_points.c b/lib/ethdev/ethdev_trace_points.c
index bd6dd4e78a..99e04f5893 100644
--- a/lib/ethdev/ethdev_trace_points.c
+++ b/lib/ethdev/ethdev_trace_points.c
@@ -777,3 +777,12 @@ RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_destroy,
 
 RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_action_list_handle_query_update,
 			 lib.ethdev.flow.async_action_list_handle_query_update)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_template_table_resize,
+			 lib.ethdev.flow.template_table_resize)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_async_update_resized,
+			 lib.ethdev.flow.async_update_resized)
+
+RTE_TRACE_POINT_REGISTER(rte_flow_trace_table_resize_complete,
+			 lib.ethdev.flow.table_resize_complete)
diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
index fb6ffe94dc..b752d05049 100644
--- a/lib/ethdev/rte_flow.c
+++ b/lib/ethdev/rte_flow.c
@@ -2590,6 +2590,83 @@ rte_flow_calc_encap_hash(uint16_t port_id, const struct rte_flow_item pattern[],
 	return flow_err(port_id, ret, error);
 }
 
+bool
+rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
+				  const struct rte_flow_template_table_attr *tbl_attr)
+{
+	return (tbl_attr->specialize &
+		RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE) != 0;
+}
+
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_resize not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize(dev, table, nb_rules, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_template_table_resize(port_id, table, nb_rules, ret);
+	return ret;
+}
+
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_update_resized)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "async_flow_async_transfer not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_update_resized(dev, queue, attr, rule, user_data, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_async_update_resized(port_id, queue, attr,
+					    rule, user_data, ret);
+	return ret;
+}
+
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error)
+{
+	int ret;
+	struct rte_eth_dev *dev;
+	const struct rte_flow_ops *ops;
+
+	RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
+	ops = rte_flow_ops_get(port_id, error);
+	if (!ops || !ops->flow_template_table_resize_complete)
+		return rte_flow_error_set(error, ENOTSUP,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					  "flow_template_table_transfer_complete not supported");
+	dev = &rte_eth_devices[port_id];
+	ret = ops->flow_template_table_resize_complete(dev, table, error);
+	ret = flow_err(port_id, ret, error);
+	rte_flow_trace_table_resize_complete(port_id, table, ret);
+	return ret;
+}
+
 static struct rte_flow *
 rte_flow_dummy_async_create(struct rte_eth_dev *dev __rte_unused,
 			    uint32_t queue __rte_unused,
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 48e4f47844..b5b58dcaa0 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -5842,6 +5842,10 @@ struct rte_flow_template_table;
  * if the hint is supported.
  */
 #define RTE_FLOW_TABLE_SPECIALIZE_TRANSFER_VPORT_ORIG RTE_BIT32(1)
+/**
+ * Specialize table for resize.
+ */
+#define RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE RTE_BIT32(2)
 /**@}*/
 
 /**
@@ -5920,6 +5924,25 @@ struct rte_flow_template_table_attr {
 	enum rte_flow_table_hash_func hash_func;
 };
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Query whether a table can be resized.
+ *
+ * @param port_id
+ *    Port identifier of Ethernet device.
+ * @param tbl_attr
+ *    Template table.
+ *
+ * @return
+ *    True if the table can be resized.
+ */
+__rte_experimental
+bool
+rte_flow_template_table_resizable(__rte_unused uint16_t port_id,
+		const struct rte_flow_template_table_attr *tbl_attr);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -6899,6 +6922,101 @@ rte_flow_calc_encap_hash(uint16_t port_id, const struct rte_flow_item pattern[],
 			 enum rte_flow_encap_hash_field dest_field, uint8_t hash_len,
 			 uint8_t *hash, struct rte_flow_error *error);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update template table for new flow rules capacity.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table to modify.
+ * @param nb_rules
+ *   New flow rules capacity.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* is not resizable or
+ *               *table* resize to *nb_rules* is not supported or
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize(uint16_t port_id,
+			       struct rte_flow_template_table *table,
+			       uint32_t nb_rules,
+			       struct rte_flow_error *error);
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Update *rule* for the new *table* configuration after table resize.
+ * Must be called for each *rule* created before *table* resize.
+ * If called for *rule* created after *table* resize returns success.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param queue
+ *   Flow queue for async operation.
+ * @param attr
+ *   Async operation attributes.
+ * @param rule
+ *   Flow rule to update.
+ * @param user_data
+ *   The user data that will be returned on async completion event.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EINVAL) if *table* was not resized.
+ *               If *rule* cannot be updated after *table* resize,
+ *               unrecoverable *table* error.
+ */
+__rte_experimental
+int
+rte_flow_async_update_resized(uint16_t port_id, uint32_t queue,
+			      const struct rte_flow_op_attr *attr,
+			      struct rte_flow *rule, void *user_data,
+			      struct rte_flow_error *error);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Resume normal operational mode after table was resized and
+ * table rules were updated for the new table configuration.
+ *
+ * @param port_id
+ *   Port identifier of Ethernet device.
+ * @param table
+ *   Template table that undergoing resize operation.
+ * @param error
+ *   Perform verbose error reporting if not NULL.
+ *   PMDs initialize this structure in case of error only.
+ *
+ * @return
+ *   - (0) if success.
+ *   - (-ENODEV) if *port_id* invalid.
+ *   - (-ENOTSUP) if underlying device does not support this functionality.
+ *   - (-EBUSY) if not all *table* rules were updated.
+ *   - (-EINVAL) if *table* cannot complete table resize,
+ *               unrecoverable error.
+ */
+__rte_experimental
+int
+rte_flow_template_table_resize_complete(uint16_t port_id,
+					struct rte_flow_template_table *table,
+					struct rte_flow_error *error);
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
index ac55b762a0..3c702e30b4 100644
--- a/lib/ethdev/rte_flow_driver.h
+++ b/lib/ethdev/rte_flow_driver.h
@@ -257,6 +257,21 @@ struct rte_flow_ops {
 		(struct rte_eth_dev *dev, const struct rte_flow_item pattern[],
 		 enum rte_flow_encap_hash_field dest_field, uint8_t *hash,
 		 struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize() */
+	int (*flow_template_table_resize)(struct rte_eth_dev *dev,
+					  struct rte_flow_template_table *table,
+					  uint32_t nb_rules,
+					  struct rte_flow_error *error);
+	/** @see rte_flow_async_update_resized() */
+	int (*flow_update_resized)(struct rte_eth_dev *dev, uint32_t queue,
+				   const struct rte_flow_op_attr *attr,
+				   struct rte_flow *rule, void *user_data,
+				   struct rte_flow_error *error);
+	/** @see rte_flow_template_table_resize_complete() */
+	int (*flow_template_table_resize_complete)
+		(struct rte_eth_dev *dev,
+		 struct rte_flow_template_table *table,
+		 struct rte_flow_error *error);
 };
 
 /**
diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
index 748dba899f..019eca209d 100644
--- a/lib/ethdev/version.map
+++ b/lib/ethdev/version.map
@@ -321,6 +321,10 @@ EXPERIMENTAL {
 	__rte_eth_trace_tx_queue_count;
 	rte_eth_find_rss_algo;
 	rte_flow_calc_encap_hash;
+	rte_flow_async_update_resized;
+	rte_flow_template_table_resizable;
+	rte_flow_template_table_resize;
+	rte_flow_template_table_resize_complete;
 };
 
 INTERNAL {
-- 
2.39.2

Re: [PATCH v8] ethdev: add template table resize API

[Ferruh Yigit]

On 2/15/2024 6:13 AM, Gregory Etelson wrote:
> Template table creation API sets table flows capacity.
> If application needs more flows then the table was designed for,
> the following procedures must be completed:
> 1. Create a new template table with larger flows capacity.
> 2. Re-create existing flows in the new table and delete flows from
>    the original table.
> 3. Destroy original table.
> 
> Application cannot always execute that procedure:
> * Port may not have sufficient resources to allocate a new table
>   while maintaining original table.
> * Application may not have existing flows "recipes" to re-create
>   flows in a new table.
> 
> The patch defines a new API that allows application to resize
> existing template table:
> 
> * Resizable template table must be created with the
>   RTE_FLOW_TABLE_SPECIALIZE_RESIZABLE_TABLE bit set.
> 
> * Application resizes existing table with the
>   `rte_flow_template_table_resize()` function call.
>   The table resize procedure updates the table maximal flow number
>   only. Other table attributes are not affected by the table resize.
>   ** The table resize procedure must not interrupt
>      existing table flows operations in hardware.
>   ** The table resize procedure must not alter flow handles held by
>      application.
> 
> * After `rte_flow_template_table_resize()` returned, application must
>   update table flow rules by calling
>   `rte_flow_async_update_resized()`.
>   The call reconfigures internal flow resources for the new table
>   configuration.
>   The flow update must not interrupt hardware flow operations.
> 
> * After table flows were updated, application must call
>   `rte_flow_template_table_resize_complete()`.
>   The function releases PMD resources related to the original
>   table.
>   Application can start new table resize after
>   `rte_flow_template_table_resize_complete()` returned.
> 
> Testpmd commands:
> 
> * Create resizable template table
> flow template_table <port-id> create table_id <tbl-id> resizable \
>   [transfer|ingress|egres] group <group-id> \
>   rules_number <initial table capacity> \
>   pattern_template  <pt1> [ pattern_template <pt2> [ ... ]] \
>   actions_template  <at1> [ actions_template <at2> [ ... ]]
> 
> * Resize table:
> flow template_table <tbl-id> resize table_resize_id <tbl-id> \
>   table_resize_rules_num <new table capacity>
> 
> * Queue a flow update:
> flow queue <port-id> update_resized <tbl-id> rule <flow-id>
> 
> * Complete table resize:
> flow template_table <port-id> resize_complete table <tbl-id>
> 
> Signed-off-by: Gregory Etelson <getelson@nvidia.com>
> Acked-by: Ori Kam <orika@nvidia.com>
> Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>
> Acked-by: Thomas Monjalon <thomas@monjalon.net>
>

Applied to dpdk-next-net/main, thanks.