RE: [RFC] ethdev: add indirect action async query

[Ori Kam <orika@nvidia.com>]

> -----Original Message-----
> From: Suanming Mou <suanmingm@nvidia.com>
> Sent: Tuesday, 9 August 2022 16:28
> Subject: [RFC] ethdev: add indirect action async query
> 
> As rte_flow_action_handle_create/destroy/update() have their own
> asynchronous rte_flow_async_action_handle_create/destroy/update()
> version functions to accelerate the indirect action operations in
> queue based flow engine. Currently, the asynchronous version query
> function for indirect action was missing.
> 
> This patch adds the rte_flow_async_action_handle_query() function
> corresponds to rte_flow_action_handle_query(). The new asynchronous
> version function enables enqueue the query to the hardware similar
> as asynchronous flow management does and returns immediately to free
> the CPU for other tasks. Application can get the query results from
> rte_flow_pull() when the hardware completes its work.
> 
> Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> ---
>  app/test-pmd/cmdline_flow.c                 |  34 +++
>  app/test-pmd/config.c                       | 240 ++++++++++++++------
>  app/test-pmd/testpmd.h                      |  28 +++
>  doc/guides/prog_guide/rte_flow.rst          |  16 ++
>  doc/guides/rel_notes/release_22_11.rst      |   5 +
>  doc/guides/testpmd_app_ug/testpmd_funcs.rst |  19 ++
>  lib/ethdev/rte_flow.c                       |  18 ++
>  lib/ethdev/rte_flow.h                       |  44 ++++
>  lib/ethdev/rte_flow_driver.h                |   9 +
>  lib/ethdev/version.map                      |   3 +
>  10 files changed, 345 insertions(+), 71 deletions(-)
> 
> diff --git a/app/test-pmd/cmdline_flow.c b/app/test-pmd/cmdline_flow.c
> index 7f50028eb7..0223286c1a 100644
> --- a/app/test-pmd/cmdline_flow.c
> +++ b/app/test-pmd/cmdline_flow.c
> @@ -145,6 +145,7 @@ enum index {
>  	QUEUE_INDIRECT_ACTION_CREATE,
>  	QUEUE_INDIRECT_ACTION_UPDATE,
>  	QUEUE_INDIRECT_ACTION_DESTROY,
> +	QUEUE_INDIRECT_ACTION_QUERY,
> 
>  	/* Queue indirect action create arguments */
>  	QUEUE_INDIRECT_ACTION_CREATE_ID,
> @@ -161,6 +162,9 @@ enum index {
>  	QUEUE_INDIRECT_ACTION_DESTROY_ID,
>  	QUEUE_INDIRECT_ACTION_DESTROY_POSTPONE,
> 
> +	/* Queue indirect action query arguments */
> +	QUEUE_INDIRECT_ACTION_QUERY_POSTPONE,
> +
>  	/* Push arguments. */
>  	PUSH_QUEUE,
> 
> @@ -1171,6 +1175,7 @@ static const enum index next_qia_subcmd[] = {
>  	QUEUE_INDIRECT_ACTION_CREATE,
>  	QUEUE_INDIRECT_ACTION_UPDATE,
>  	QUEUE_INDIRECT_ACTION_DESTROY,
> +	QUEUE_INDIRECT_ACTION_QUERY,
>  	ZERO,
>  };
> 
> @@ -1197,6 +1202,12 @@ static const enum index next_qia_destroy_attr[] =
> {
>  	ZERO,
>  };
> 
> +static const enum index next_qia_query_attr[] = {
> +	QUEUE_INDIRECT_ACTION_QUERY_POSTPONE,
> +	END,
> +	ZERO,
> +};
> +
>  static const enum index next_ia_create_attr[] = {
>  	INDIRECT_ACTION_CREATE_ID,
>  	INDIRECT_ACTION_INGRESS,
> @@ -3013,6 +3024,14 @@ static const struct token token_list[] = {
>  		.next = NEXT(next_qia_destroy_attr),
>  		.call = parse_qia_destroy,
>  	},
> +	[QUEUE_INDIRECT_ACTION_QUERY] = {
> +		.name = "query",
> +		.help = "query indirect action",
> +		.next = NEXT(next_qia_query_attr,
> +			     NEXT_ENTRY(COMMON_INDIRECT_ACTION_ID)),
> +		.args = ARGS(ARGS_ENTRY(struct buffer, args.vc.attr.group)),
> +		.call = parse_qia,
> +	},
>  	/* Indirect action destroy arguments. */
>  	[QUEUE_INDIRECT_ACTION_DESTROY_POSTPONE] = {
>  		.name = "postpone",
> @@ -3038,6 +3057,14 @@ static const struct token token_list[] = {
>  			     NEXT_ENTRY(COMMON_BOOLEAN)),
>  		.args = ARGS(ARGS_ENTRY(struct buffer, postpone)),
>  	},
> +	/* Indirect action update arguments. */
> +	[QUEUE_INDIRECT_ACTION_QUERY_POSTPONE] = {
> +		.name = "postpone",
> +		.help = "postpone query operation",
> +		.next = NEXT(next_qia_query_attr,
> +			     NEXT_ENTRY(COMMON_BOOLEAN)),
> +		.args = ARGS(ARGS_ENTRY(struct buffer, postpone)),
> +	},
>  	/* Indirect action create arguments. */
>  	[QUEUE_INDIRECT_ACTION_CREATE_ID] = {
>  		.name = "action_id",
> @@ -6682,6 +6709,8 @@ parse_qia(struct context *ctx, const struct token
> *token,
>  			(void *)RTE_ALIGN_CEIL((uintptr_t)(out + 1),
>  					       sizeof(double));
>  		out->args.vc.attr.group = UINT32_MAX;
> +		/* fallthrough */
> +	case QUEUE_INDIRECT_ACTION_QUERY:
>  		out->command = ctx->curr;
>  		ctx->objdata = 0;
>  		ctx->object = out;
> @@ -10509,6 +10538,11 @@ cmd_flow_parsed(const struct buffer *in)
>  						in->args.vc.attr.group,
>  						in->args.vc.actions);
>  		break;
> +	case QUEUE_INDIRECT_ACTION_QUERY:
> +		port_queue_action_handle_query(in->port,
> +					       in->queue, in->postpone,
> +					       in->args.vc.attr.group);
> +		break;
>  	case INDIRECT_ACTION_CREATE:
>  		port_action_handle_create(
>  				in->port, in->args.vc.attr.group,
> diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
> index a2939867c4..4c51ed03a8 100644
> --- a/app/test-pmd/config.c
> +++ b/app/test-pmd/config.c
> @@ -2080,44 +2080,18 @@ port_action_handle_update(portid_t port_id,
> uint32_t id,
>  	return 0;
>  }
> 
> -int
> -port_action_handle_query(portid_t port_id, uint32_t id)
> +static void
> +port_action_handle_query_dump(uint32_t type, union port_action_query
> *query)
>  {
> -	struct rte_flow_error error;
> -	struct port_indirect_action *pia;
> -	union {
> -		struct rte_flow_query_count count;
> -		struct rte_flow_query_age age;
> -		struct rte_flow_action_conntrack ct;
> -	} query;
> -
> -	pia = action_get_by_id(port_id, id);
> -	if (!pia)
> -		return -EINVAL;
> -	switch (pia->type) {
> -	case RTE_FLOW_ACTION_TYPE_AGE:
> -	case RTE_FLOW_ACTION_TYPE_COUNT:
> -		break;
> -	default:
> -		fprintf(stderr,
> -			"Indirect action %u (type: %d) on port %u doesn't
> support query\n",
> -			id, pia->type, port_id);
> -		return -ENOTSUP;
> -	}
> -	/* Poisoning to make sure PMDs update it in case of error. */
> -	memset(&error, 0x55, sizeof(error));
> -	memset(&query, 0, sizeof(query));
> -	if (rte_flow_action_handle_query(port_id, pia->handle, &query,
> &error))
> -		return port_flow_complain(&error);
> -	switch (pia->type) {
> +	switch (type) {
>  	case RTE_FLOW_ACTION_TYPE_AGE:
>  		printf("Indirect AGE action:\n"
>  		       " aged: %u\n"
>  		       " sec_since_last_hit_valid: %u\n"
>  		       " sec_since_last_hit: %" PRIu32 "\n",
> -		       query.age.aged,
> -		       query.age.sec_since_last_hit_valid,
> -		       query.age.sec_since_last_hit);
> +		       query->age.aged,
> +		       query->age.sec_since_last_hit_valid,
> +		       query->age.sec_since_last_hit);
>  		break;
>  	case RTE_FLOW_ACTION_TYPE_COUNT:
>  		printf("Indirect COUNT action:\n"
> @@ -2125,10 +2099,10 @@ port_action_handle_query(portid_t port_id,
> uint32_t id)
>  		       " bytes_set: %u\n"
>  		       " hits: %" PRIu64 "\n"
>  		       " bytes: %" PRIu64 "\n",
> -		       query.count.hits_set,
> -		       query.count.bytes_set,
> -		       query.count.hits,
> -		       query.count.bytes);
> +		       query->count.hits_set,
> +		       query->count.bytes_set,
> +		       query->count.hits,
> +		       query->count.bytes);
>  		break;
>  	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
>  		printf("Conntrack Context:\n"
> @@ -2138,47 +2112,76 @@ port_action_handle_query(portid_t port_id,
> uint32_t id)
>  		       "  Factor: %u, Retrans: %u, TCP flags: %u\n"
>  		       "  Last Seq: %u, Last ACK: %u\n"
>  		       "  Last Win: %u, Last End: %u\n",
> -		       query.ct.peer_port,
> -		       query.ct.is_original_dir ? "Original" : "Reply",
> -		       query.ct.enable, query.ct.live_connection,
> -		       query.ct.selective_ack, query.ct.challenge_ack_passed,
> -		       query.ct.last_direction ? "Original" : "Reply",
> -		       query.ct.liberal_mode, query.ct.state,
> -		       query.ct.max_ack_window,
> query.ct.retransmission_limit,
> -		       query.ct.last_index, query.ct.last_seq,
> -		       query.ct.last_ack, query.ct.last_window,
> -		       query.ct.last_end);
> +		       query->ct.peer_port,
> +		       query->ct.is_original_dir ? "Original" : "Reply",
> +		       query->ct.enable, query->ct.live_connection,
> +		       query->ct.selective_ack, query-
> >ct.challenge_ack_passed,
> +		       query->ct.last_direction ? "Original" : "Reply",
> +		       query->ct.liberal_mode, query->ct.state,
> +		       query->ct.max_ack_window, query-
> >ct.retransmission_limit,
> +		       query->ct.last_index, query->ct.last_seq,
> +		       query->ct.last_ack, query->ct.last_window,
> +		       query->ct.last_end);
>  		printf("  Original Dir:\n"
>  		       "    scale: %u, fin: %u, ack seen: %u\n"
>  		       " unacked data: %u\n    Sent end: %u,"
>  		       "    Reply end: %u, Max win: %u, Max ACK: %u\n",
> -		       query.ct.original_dir.scale,
> -		       query.ct.original_dir.close_initiated,
> -		       query.ct.original_dir.last_ack_seen,
> -		       query.ct.original_dir.data_unacked,
> -		       query.ct.original_dir.sent_end,
> -		       query.ct.original_dir.reply_end,
> -		       query.ct.original_dir.max_win,
> -		       query.ct.original_dir.max_ack);
> +		       query->ct.original_dir.scale,
> +		       query->ct.original_dir.close_initiated,
> +		       query->ct.original_dir.last_ack_seen,
> +		       query->ct.original_dir.data_unacked,
> +		       query->ct.original_dir.sent_end,
> +		       query->ct.original_dir.reply_end,
> +		       query->ct.original_dir.max_win,
> +		       query->ct.original_dir.max_ack);
>  		printf("  Reply Dir:\n"
>  		       "    scale: %u, fin: %u, ack seen: %u\n"
>  		       " unacked data: %u\n    Sent end: %u,"
>  		       "    Reply end: %u, Max win: %u, Max ACK: %u\n",
> -		       query.ct.reply_dir.scale,
> -		       query.ct.reply_dir.close_initiated,
> -		       query.ct.reply_dir.last_ack_seen,
> -		       query.ct.reply_dir.data_unacked,
> -		       query.ct.reply_dir.sent_end,
> -		       query.ct.reply_dir.reply_end,
> -		       query.ct.reply_dir.max_win,
> -		       query.ct.reply_dir.max_ack);
> +		       query->ct.reply_dir.scale,
> +		       query->ct.reply_dir.close_initiated,
> +		       query->ct.reply_dir.last_ack_seen,
> +		       query->ct.reply_dir.data_unacked,
> +		       query->ct.reply_dir.sent_end,
> +		       query->ct.reply_dir.reply_end,
> +		       query->ct.reply_dir.max_win,
> +		       query->ct.reply_dir.max_ack);
> +		break;
> +	default:
> +		fprintf(stderr,
> +			"Indirect action (type: %d) doesn't support query\n",
> +			type);
> +		break;
> +	}
> +
> +}
> +
> +int
> +port_action_handle_query(portid_t port_id, uint32_t id)
> +{
> +	struct rte_flow_error error;
> +	struct port_indirect_action *pia;
> +	union port_action_query query;
> +
> +	pia = action_get_by_id(port_id, id);
> +	if (!pia)
> +		return -EINVAL;
> +	switch (pia->type) {
> +	case RTE_FLOW_ACTION_TYPE_AGE:
> +	case RTE_FLOW_ACTION_TYPE_COUNT:
>  		break;
>  	default:
>  		fprintf(stderr,
>  			"Indirect action %u (type: %d) on port %u doesn't
> support query\n",
>  			id, pia->type, port_id);
> -		break;
> +		return -ENOTSUP;
>  	}
> +	/* Poisoning to make sure PMDs update it in case of error. */
> +	memset(&error, 0x55, sizeof(error));
> +	memset(&query, 0, sizeof(query));
> +	if (rte_flow_action_handle_query(port_id, pia->handle, &query,
> &error))
> +		return port_flow_complain(&error);
> +	port_action_handle_query_dump(pia->type, &query);
>  	return 0;
>  }
> 
> @@ -2670,6 +2673,7 @@ port_queue_flow_create(portid_t port_id,
> queueid_t queue_id,
>  	bool found;
>  	struct rte_flow_error error = { RTE_FLOW_ERROR_TYPE_NONE,
> NULL, NULL };
>  	struct rte_flow_action_age *age = age_action_get(actions);
> +	struct queue_job *job;
> 
>  	port = &ports[port_id];
>  	if (port->flow_list) {
> @@ -2713,9 +2717,18 @@ port_queue_flow_create(portid_t port_id,
> queueid_t queue_id,
>  		return -EINVAL;
>  	}
> 
> +	job = calloc(1, sizeof(*job));
> +	if (!job) {
> +		printf("Queue flow create job allocate failed\n");
> +		return -ENOMEM;
> +	}
> +	job->type = QUEUE_JOB_TYPE_FLOW_CREATE;
> +
>  	pf = port_flow_new(NULL, pattern, actions, &error);
> -	if (!pf)
> +	if (!pf) {
> +		free(job);
>  		return port_flow_complain(&error);
> +	}
>  	if (age) {
>  		pf->age_type = ACTION_AGE_CONTEXT_TYPE_FLOW;
>  		age->context = &pf->age_type;
> @@ -2723,16 +2736,18 @@ port_queue_flow_create(portid_t port_id,
> queueid_t queue_id,
>  	/* Poisoning to make sure PMDs update it in case of error. */
>  	memset(&error, 0x11, sizeof(error));
>  	flow = rte_flow_async_create(port_id, queue_id, &op_attr, pt-
> >table,
> -		pattern, pattern_idx, actions, actions_idx, NULL, &error);
> +		pattern, pattern_idx, actions, actions_idx, job, &error);
>  	if (!flow) {
>  		uint32_t flow_id = pf->id;
>  		port_queue_flow_destroy(port_id, queue_id, true, 1,
> &flow_id);
> +		free(job);
>  		return port_flow_complain(&error);
>  	}
> 
>  	pf->next = port->flow_list;
>  	pf->id = id;
>  	pf->flow = flow;
> +	job->pf = pf;
>  	port->flow_list = pf;
>  	printf("Flow rule #%u creation enqueued\n", pf->id);
>  	return 0;
> @@ -2748,6 +2763,7 @@ port_queue_flow_destroy(portid_t port_id,
> queueid_t queue_id,
>  	struct port_flow **tmp;
>  	uint32_t c = 0;
>  	int ret = 0;
> +	struct queue_job *job;
> 
>  	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
>  	    port_id == (portid_t)RTE_PORT_ALL)
> @@ -2774,14 +2790,22 @@ port_queue_flow_destroy(portid_t port_id,
> queueid_t queue_id,
>  			 * update it in case of error.
>  			 */
>  			memset(&error, 0x33, sizeof(error));
> +			job = calloc(1, sizeof(*job));
> +			if (!job) {
> +				printf("Queue flow destroy job allocate
> failed\n");
> +				return -ENOMEM;
> +			}
> +			job->type = QUEUE_JOB_TYPE_FLOW_DESTROY;
> +			job->pf = pf;
> +
>  			if (rte_flow_async_destroy(port_id, queue_id,
> &op_attr,
> -						   pf->flow, NULL, &error)) {
> +						   pf->flow, job, &error)) {
> +				free(job);
>  				ret = port_flow_complain(&error);
>  				continue;
>  			}
>  			printf("Flow rule #%u destruction enqueued\n", pf-
> >id);
>  			*tmp = pf->next;
> -			free(pf);
>  			break;
>  		}
>  		if (i == n)
> @@ -2803,6 +2827,7 @@ port_queue_action_handle_create(portid_t
> port_id, uint32_t queue_id,
>  	struct port_indirect_action *pia;
>  	int ret;
>  	struct rte_flow_error error;
> +	struct queue_job *job;
> 
>  	ret = action_alloc(port_id, id, &pia);
>  	if (ret)
> @@ -2813,6 +2838,13 @@ port_queue_action_handle_create(portid_t
> port_id, uint32_t queue_id,
>  		printf("Queue #%u is invalid\n", queue_id);
>  		return -EINVAL;
>  	}
> +	job = calloc(1, sizeof(*job));
> +	if (!job) {
> +		printf("Queue action create job allocate failed\n");
> +		return -ENOMEM;
> +	}
> +	job->type = QUEUE_JOB_TYPE_ACTION_CREATE;
> +	job->pia = pia;
> 
>  	if (action->type == RTE_FLOW_ACTION_TYPE_AGE) {
>  		struct rte_flow_action_age *age =
> @@ -2824,11 +2856,12 @@ port_queue_action_handle_create(portid_t
> port_id, uint32_t queue_id,
>  	/* Poisoning to make sure PMDs update it in case of error. */
>  	memset(&error, 0x88, sizeof(error));
>  	pia->handle = rte_flow_async_action_handle_create(port_id,
> queue_id,
> -					&attr, conf, action, NULL, &error);
> +					&attr, conf, action, job, &error);
>  	if (!pia->handle) {
>  		uint32_t destroy_id = pia->id;
>  		port_queue_action_handle_destroy(port_id, queue_id,
>  						 postpone, 1, &destroy_id);
> +		free(job);
>  		return port_flow_complain(&error);
>  	}
>  	pia->type = action->type;
> @@ -2847,6 +2880,7 @@ port_queue_action_handle_destroy(portid_t
> port_id,
>  	struct port_indirect_action **tmp;
>  	uint32_t c = 0;
>  	int ret = 0;
> +	struct queue_job *job;
> 
>  	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
>  	    port_id == (portid_t)RTE_PORT_ALL)
> @@ -2873,17 +2907,23 @@ port_queue_action_handle_destroy(portid_t
> port_id,
>  			 * of error.
>  			 */
>  			memset(&error, 0x99, sizeof(error));
> +			job = calloc(1, sizeof(*job));
> +			if (!job) {
> +				printf("Queue action destroy job allocate
> failed\n");
> +				return -ENOMEM;
> +			}
> +			job->type = QUEUE_JOB_TYPE_ACTION_DESTROY;
> +			job->pia = pia;
> 
>  			if (pia->handle &&
>  			    rte_flow_async_action_handle_destroy(port_id,
> -				queue_id, &attr, pia->handle, NULL, &error))
> {
> +				queue_id, &attr, pia->handle, job, &error)) {
>  				ret = port_flow_complain(&error);
>  				continue;
>  			}
>  			*tmp = pia->next;
>  			printf("Indirect action #%u destruction queued\n",
>  			       pia->id);
> -			free(pia);
>  			break;
>  		}
>  		if (i == n)
> @@ -2903,6 +2943,7 @@ port_queue_action_handle_update(portid_t
> port_id,
>  	struct rte_port *port;
>  	struct rte_flow_error error;
>  	struct rte_flow_action_handle *action_handle;
> +	struct queue_job *job;
> 
>  	action_handle = port_action_handle_get_by_id(port_id, id);
>  	if (!action_handle)
> @@ -2914,8 +2955,56 @@ port_queue_action_handle_update(portid_t
> port_id,
>  		return -EINVAL;
>  	}
> 
> +	job = calloc(1, sizeof(*job));
> +	if (!job) {
> +		printf("Queue action update job allocate failed\n");
> +		return -ENOMEM;
> +	}
> +	job->type = QUEUE_JOB_TYPE_ACTION_UPDATE;
> +
>  	if (rte_flow_async_action_handle_update(port_id, queue_id, &attr,
> -				    action_handle, action, NULL, &error)) {
> +				    action_handle, action, job, &error)) {
> +		free(job);
> +		return port_flow_complain(&error);
> +	}
> +	printf("Indirect action #%u update queued\n", id);
> +	return 0;
> +}
> +
> +/** Enqueue indirect action query operation. */
> +int
> +port_queue_action_handle_query(portid_t port_id,
> +			       uint32_t queue_id, bool postpone, uint32_t id)
> +{
> +	const struct rte_flow_op_attr attr = { .postpone = postpone};
> +	struct rte_port *port;
> +	struct rte_flow_error error;
> +	struct rte_flow_action_handle *action_handle;
> +	struct port_indirect_action *pia;
> +	struct queue_job *job;
> +
> +	pia = action_get_by_id(port_id, id);
> +	action_handle = pia ? pia->handle : NULL;
> +	if (!action_handle)
> +		return -EINVAL;
> +
> +	port = &ports[port_id];
> +	if (queue_id >= port->queue_nb) {
> +		printf("Queue #%u is invalid\n", queue_id);
> +		return -EINVAL;
> +	}
> +
> +	job = calloc(1, sizeof(*job));
> +	if (!job) {
> +		printf("Queue action update job allocate failed\n");
> +		return -ENOMEM;
> +	}
> +	job->type = QUEUE_JOB_TYPE_ACTION_QUERY;
> +	job->pia = pia;
> +
> +	if (rte_flow_async_action_handle_query(port_id, queue_id, &attr,
> +				    action_handle, &job->query, job, &error)) {
> +		free(job);
>  		return port_flow_complain(&error);
>  	}
>  	printf("Indirect action #%u update queued\n", id);
> @@ -2960,6 +3049,7 @@ port_queue_flow_pull(portid_t port_id, queueid_t
> queue_id)
>  	int ret = 0;
>  	int success = 0;
>  	int i;
> +	struct queue_job *job;
> 
>  	if (port_id_is_invalid(port_id, ENABLED_WARN) ||
>  	    port_id == (portid_t)RTE_PORT_ALL)
> @@ -2989,6 +3079,14 @@ port_queue_flow_pull(portid_t port_id,
> queueid_t queue_id)
>  	for (i = 0; i < ret; i++) {
>  		if (res[i].status == RTE_FLOW_OP_SUCCESS)
>  			success++;
> +		job = (struct queue_job *)res[i].user_data;
> +		if (job->type == QUEUE_JOB_TYPE_FLOW_DESTROY)
> +			free(job->pf);
> +		else if (job->type == QUEUE_JOB_TYPE_ACTION_DESTROY)
> +			free(job->pia);
> +		else if (job->type == QUEUE_JOB_TYPE_ACTION_QUERY)
> +			port_action_handle_query_dump(job->pia->type,
> &job->query);
> +		free(job);
>  	}
>  	printf("Queue #%u pulled %u operations (%u failed, %u
> succeeded)\n",
>  	       queue_id, ret, ret - success, success);
> diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
> index fb2f5195d3..c7a96d062c 100644
> --- a/app/test-pmd/testpmd.h
> +++ b/app/test-pmd/testpmd.h
> @@ -105,6 +105,15 @@ enum {
>  	/**< allocate mempool natively, use
> rte_pktmbuf_pool_create_extbuf */
>  };
> 
> +enum {
> +	QUEUE_JOB_TYPE_FLOW_CREATE,
> +	QUEUE_JOB_TYPE_FLOW_DESTROY,
> +	QUEUE_JOB_TYPE_ACTION_CREATE,
> +	QUEUE_JOB_TYPE_ACTION_DESTROY,
> +	QUEUE_JOB_TYPE_ACTION_UPDATE,
> +	QUEUE_JOB_TYPE_ACTION_QUERY,
> +};
> +
>  /**
>   * The data structure associated with RX and TX packet burst statistics
>   * that are recorded for each forwarding stream.
> @@ -220,6 +229,23 @@ struct port_indirect_action {
>  	enum age_action_context_type age_type; /**< Age action context
> type. */
>  };
> 
> +/* Descriptor for action query data. */
> +union port_action_query {
> +	struct rte_flow_query_count count;
> +	struct rte_flow_query_age age;
> +	struct rte_flow_action_conntrack ct;
> +};
> +
> +/* Descriptor for queue job. */
> +struct queue_job {
> +	uint32_t type; /**< Job type. */
> +	union {
> +		struct port_flow *pf;
> +		struct port_indirect_action *pia;
> +	};
> +	union port_action_query query;
> +};
> +
>  struct port_flow_tunnel {
>  	LIST_ENTRY(port_flow_tunnel) chain;
>  	struct rte_flow_action *pmd_actions;
> @@ -980,6 +1006,8 @@ int port_queue_action_handle_destroy(portid_t
> port_id,
>  int port_queue_action_handle_update(portid_t port_id, uint32_t queue_id,
>  				    bool postpone, uint32_t id,
>  				    const struct rte_flow_action *action);
> +int port_queue_action_handle_query(portid_t port_id, uint32_t queue_id,
> +				   bool postpone, uint32_t id);
>  int port_queue_flow_push(portid_t port_id, queueid_t queue_id);
>  int port_queue_flow_pull(portid_t port_id, queueid_t queue_id);
>  int port_flow_validate(portid_t port_id,
> diff --git a/doc/guides/prog_guide/rte_flow.rst
> b/doc/guides/prog_guide/rte_flow.rst
> index 588914b231..9e6aadf954 100644
> --- a/doc/guides/prog_guide/rte_flow.rst
> +++ b/doc/guides/prog_guide/rte_flow.rst
> @@ -3911,6 +3911,22 @@ Asynchronous version of indirect action update
> API.
>             void *user_data,
>             struct rte_flow_error *error);
> 
> +Enqueue indirect action query operation
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +Asynchronous version of indirect action query API.
> +
> +.. code-block:: c
> +
> +   int
> +   rte_flow_async_action_handle_query(uint16_t port_id,
> +           uint32_t queue_id,
> +           const struct rte_flow_op_attr *q_ops_attr,
> +           struct rte_flow_action_handle *action_handle,
> +           void *data,
> +           void *user_data,
> +           struct rte_flow_error *error);
> +
>  Push enqueued operations
>  ~~~~~~~~~~~~~~~~~~~~~~~~
> 
> diff --git a/doc/guides/rel_notes/release_22_11.rst
> b/doc/guides/rel_notes/release_22_11.rst
> index 8c021cf050..597b28ede1 100644
> --- a/doc/guides/rel_notes/release_22_11.rst
> +++ b/doc/guides/rel_notes/release_22_11.rst
> @@ -55,6 +55,11 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
> 
> +* **Added support for queue based async query in rte_flow.**
> +
> +  Added new API ``rte_flow_async_action_handle_query()``, to query the
> +  action asynchronously.
> +
> 
>  Removed Items
>  -------------
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 330e34427d..d2c6e385db 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -4676,6 +4676,25 @@ Query indirect action having id 100::
> 
>     testpmd> flow indirect_action 0 query 100
> 
> +Enqueueing query of indirect actions
> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +``flow queue indirect_action query`` adds query operation for an indirect
> +action to a queue. It is bound to
> ``rte_flow_async_action_handle_query()``::
> +
> +   flow queue {port_id} indirect_action {queue_id} query
> +      {indirect_action_id} [postpone {boolean}]
> +
> +If successful, it will show::
> +
> +   Indirect action #[...] query queued
> +
> +Otherwise it will show an error message of the form::
> +
> +   Caught error type [...] ([...]): [...]
> +
> +``flow queue pull`` must be called to retrieve the operation status.
> +
>  Sample QinQ flow rules
>  ~~~~~~~~~~~~~~~~~~~~~~
> 
> diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
> index 501be9d602..eb6a6b737e 100644
> --- a/lib/ethdev/rte_flow.c
> +++ b/lib/ethdev/rte_flow.c
> @@ -1844,3 +1844,21 @@ rte_flow_async_action_handle_update(uint16_t
> port_id,
>  					  action_handle, update, user_data,
> error);
>  	return flow_err(port_id, ret, error);
>  }
> +
> +int
> +rte_flow_async_action_handle_query(uint16_t port_id,
> +		uint32_t queue_id,
> +		const struct rte_flow_op_attr *op_attr,
> +		const struct rte_flow_action_handle *action_handle,
> +		void *data,
> +		void *user_data,
> +		struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +	int ret;
> +
> +	ret = ops->async_action_handle_query(dev, queue_id, op_attr,
> +					  action_handle, data, user_data,
> error);
> +	return flow_err(port_id, ret, error);
> +}
> diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
> index a79f1e7ef0..a5e84bd85e 100644
> --- a/lib/ethdev/rte_flow.h
> +++ b/lib/ethdev/rte_flow.h
> @@ -5612,6 +5612,50 @@ rte_flow_async_action_handle_update(uint16_t
> port_id,
>  		const void *update,
>  		void *user_data,
>  		struct rte_flow_error *error);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Enqueue indirect action query operation.
> + *
> + * Retrieve action-specific data such as counters.
> + * Data is gathered by special action which may be present/referenced in
> + * more than one flow rule definition.
> + * Data will be available only when completion event returns.
> + *
> + * @see RTE_FLOW_ACTION_TYPE_CONNTRACK
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] queue_id
> + *   Flow queue which is used to query the action.
> + * @param[in] op_attr
> + *   Indirect action update operation attributes.
> + * @param[in] action_handle
> + *   Handle for the action object to query.
> + * @param[in, out] data
> + *   Pointer to storage for the associated query data type.
> + *   The out data will be available only when completion event returns
> + *   from rte_flow_pull.
> + * @param[in] user_data
> + *   The user data that will be returned on the completion events.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL. PMDs initialize this
> + *   structure in case of error only.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +int
> +rte_flow_async_action_handle_query(uint16_t port_id,
> +		uint32_t queue_id,
> +		const struct rte_flow_op_attr *op_attr,
> +		const struct rte_flow_action_handle *action_handle,
> +		void *data,
> +		void *user_data,
> +		struct rte_flow_error *error);
>  #ifdef __cplusplus
>  }
>  #endif
> diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
> index 2bff732d6a..7289deb538 100644
> --- a/lib/ethdev/rte_flow_driver.h
> +++ b/lib/ethdev/rte_flow_driver.h
> @@ -260,6 +260,15 @@ struct rte_flow_ops {
>  		 const void *update,
>  		 void *user_data,
>  		 struct rte_flow_error *error);
> +	/** See rte_flow_async_action_handle_query() */
> +	int (*async_action_handle_query)
> +		(struct rte_eth_dev *dev,
> +		 uint32_t queue_id,
> +		 const struct rte_flow_op_attr *op_attr,
> +		 const struct rte_flow_action_handle *action_handle,
> +		 void *data,
> +		 void *user_data,
> +		 struct rte_flow_error *error);
>  };
> 
>  /**
> diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
> index 03f52fee91..722081a8c7 100644
> --- a/lib/ethdev/version.map
> +++ b/lib/ethdev/version.map
> @@ -285,6 +285,9 @@ EXPERIMENTAL {
>  	rte_mtr_color_in_protocol_priority_get;
>  	rte_mtr_color_in_protocol_set;
>  	rte_mtr_meter_vlan_table_update;
> +
> +	# added in 22.11
> +	rte_flow_async_action_handle_query;
>  };
> 
>  INTERNAL {
> --
> 2.25.1

Acked-by: Ori Kam <orika@nvidia.com>
Best,
Ori