Re: [PATCH v5 02/10] ethdev: add flow item/action templates

[Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>]

On 2/11/22 05:26, Alexander Kozyrev wrote:
> Treating every single flow rule as a completely independent and separate
> entity negatively impacts the flow rules insertion rate. Oftentimes in an
> application, many flow rules share a common structure (the same item mask
> and/or action list) so they can be grouped and classified together.
> This knowledge may be used as a source of optimization by a PMD/HW.
> 
> The pattern template defines common matching fields (the item mask) without
> values. The actions template holds a list of action types that will be used
> together in the same rule. The specific values for items and actions will
> be given only during the rule creation.
> 
> A table combines pattern and actions templates along with shared flow rule
> attributes (group ID, priority and traffic direction). This way a PMD/HW
> can prepare all the resources needed for efficient flow rules creation in
> the datapath. To avoid any hiccups due to memory reallocation, the maximum
> number of flow rules is defined at the table creation time.
> 
> The flow rule creation is done by selecting a table, a pattern template
> and an actions template (which are bound to the table), and setting unique
> values for the items and actions.
> 
> Signed-off-by: Alexander Kozyrev <akozyrev@nvidia.com>
> Acked-by: Ori Kam <orika@nvidia.com>
> ---
>   doc/guides/prog_guide/rte_flow.rst     | 124 ++++++++++++
>   doc/guides/rel_notes/release_22_03.rst |   8 +
>   lib/ethdev/rte_flow.c                  | 147 ++++++++++++++
>   lib/ethdev/rte_flow.h                  | 260 +++++++++++++++++++++++++
>   lib/ethdev/rte_flow_driver.h           |  37 ++++
>   lib/ethdev/version.map                 |   6 +
>   6 files changed, 582 insertions(+)
> 
> diff --git a/doc/guides/prog_guide/rte_flow.rst b/doc/guides/prog_guide/rte_flow.rst
> index 72fb1132ac..5391648833 100644
> --- a/doc/guides/prog_guide/rte_flow.rst
> +++ b/doc/guides/prog_guide/rte_flow.rst
> @@ -3626,6 +3626,130 @@ of pre-configurable resources for a given port on a system.
>                        struct rte_flow_port_info *port_info,
>                        struct rte_flow_error *error);
>   
> +Flow templates
> +~~~~~~~~~~~~~~
> +
> +Oftentimes in an application, many flow rules share a common structure
> +(the same pattern and/or action list) so they can be grouped and classified
> +together. This knowledge may be used as a source of optimization by a PMD/HW.
> +The flow rule creation is done by selecting a table, a pattern template
> +and an actions template (which are bound to the table), and setting unique
> +values for the items and actions. This API is not thread-safe.
> +
> +Pattern templates
> +^^^^^^^^^^^^^^^^^
> +
> +The pattern template defines a common pattern (the item mask) without values.
> +The mask value is used to select a field to match on, spec/last are ignored.
> +The pattern template may be used by multiple tables and must not be destroyed
> +until all these tables are destroyed first.
> +
> +.. code-block:: c
> +
> +	struct rte_flow_pattern_template *
> +	rte_flow_pattern_template_create(uint16_t port_id,
> +				const struct rte_flow_pattern_template_attr *template_attr,
> +				const struct rte_flow_item pattern[],
> +				struct rte_flow_error *error);
> +
> +For example, to create a pattern template to match on the destination MAC:
> +
> +.. code-block:: c
> +
> +	struct rte_flow_item pattern[2] = {{0}};
> +	struct rte_flow_item_eth eth_m = {0};
> +	pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
> +	eth_m.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff";
> +	pattern[0].mask = &eth_m;
> +	pattern[1].type = RTE_FLOW_ITEM_TYPE_END;
> +
> +	struct rte_flow_pattern_template *pattern_template =
> +		rte_flow_pattern_template_create(port, &itr, &pattern, &error);

itr?

> +
> +The concrete value to match on will be provided at the rule creation.
> +
> +Actions templates
> +^^^^^^^^^^^^^^^^^
> +
> +The actions template holds a list of action types to be used in flow rules.
> +The mask parameter allows specifying a shared constant value for every rule.
> +The actions template may be used by multiple tables and must not be destroyed
> +until all these tables are destroyed first.
> +
> +.. code-block:: c
> +
> +	struct rte_flow_actions_template *
> +	rte_flow_actions_template_create(uint16_t port_id,
> +				const struct rte_flow_actions_template_attr *template_attr,
> +				const struct rte_flow_action actions[],
> +				const struct rte_flow_action masks[],
> +				struct rte_flow_error *error);
> +
> +For example, to create an actions template with the same Mark ID
> +but different Queue Index for every rule:
> +
> +.. code-block:: c
> +
> +	struct rte_flow_action actions[] = {
> +		/* Mark ID is constant (4) for every rule, Queue Index is unique */
> +		[0] = {.type = RTE_FLOW_ACTION_TYPE_MARK,
> +			   .conf = &(struct rte_flow_action_mark){.id = 4}},
> +		[1] = {.type = RTE_FLOW_ACTION_TYPE_QUEUE},
> +		[2] = {.type = RTE_FLOW_ACTION_TYPE_END,},
> +	};
> +	struct rte_flow_action masks[] = {
> +		/* Assign to MARK mask any non-zero value to make it constant */
> +		[0] = {.type = RTE_FLOW_ACTION_TYPE_MARK,
> +			   .conf = &(struct rte_flow_action_mark){.id = 1}},
> +		[1] = {.type = RTE_FLOW_ACTION_TYPE_QUEUE},
> +		[2] = {.type = RTE_FLOW_ACTION_TYPE_END,},
> +	};
> +
> +	struct rte_flow_actions_template *at =
> +		rte_flow_actions_template_create(port, &atr, &actions, &masks, &error);

atr?

> +
> +The concrete value for Queue Index will be provided at the rule creation.
> +
> +Template table
> +^^^^^^^^^^^^^^
> +
> +A template table combines a number of pattern and actions templates along with
> +shared flow rule attributes (group ID, priority and traffic direction).
> +This way a PMD/HW can prepare all the resources needed for efficient flow rules
> +creation in the datapath. To avoid any hiccups due to memory reallocation,
> +the maximum number of flow rules is defined at table creation time.
> +Any flow rule creation beyond the maximum table size is rejected.
> +Application may create another table to accommodate more rules in this case.
> +
> +.. code-block:: c
> +
> +	struct rte_flow_template_table *
> +	rte_flow_template_table_create(uint16_t port_id,
> +				const struct rte_flow_template_table_attr *table_attr,
> +				struct rte_flow_pattern_template *pattern_templates[],

const?

> +				uint8_t nb_pattern_templates,
> +				struct rte_flow_actions_template *actions_templates[],

const?

> +				uint8_t nb_actions_templates,
> +				struct rte_flow_error *error);
> +
> +A table can be created only after the Flow Rules management is configured
> +and pattern and actions templates are created.
> +
> +.. code-block:: c
> +
> +	rte_flow_configure(port, *port_attr, *error);


Why do you have '*' before port_attr and error above?


> +
> +	struct rte_flow_pattern_template *pattern_templates[0] =

Definition of zero size array looks wrong.

> +		rte_flow_pattern_template_create(port, &itr, &pattern, &error);

itr?

> +	struct rte_flow_actions_template *actions_templates[0] =

Zero size array?

> +		rte_flow_actions_template_create(port, &atr, &actions, &masks, &error);

atr?

> +
> +	struct rte_flow_template_table *table =
> +		rte_flow_template_table_create(port, *table_attr,
> +				*pattern_templates, nb_pattern_templates,
> +				*actions_templates, nb_actions_templates,
> +				*error);

Similar question here.

> +
>   .. _flow_isolated_mode:
>   
>   Flow isolated mode
> diff --git a/doc/guides/rel_notes/release_22_03.rst b/doc/guides/rel_notes/release_22_03.rst
> index 2a47a37f0a..6656b35295 100644
> --- a/doc/guides/rel_notes/release_22_03.rst
> +++ b/doc/guides/rel_notes/release_22_03.rst
> @@ -75,6 +75,14 @@ New Features
>       engine, allowing to pre-allocate some resources for better performance.
>       Added ``rte_flow_info_get`` API to retrieve pre-configurable resources.
>   
> +  * ethdev: Added ``rte_flow_template_table_create`` API to group flow rules
> +    with the same flow attributes and common matching patterns and actions
> +    defined by ``rte_flow_pattern_template_create`` and
> +    ``rte_flow_actions_template_create`` respectively.
> +    Corresponding functions to destroy these entities are:
> +    ``rte_flow_template_table_destroy``, ``rte_flow_pattern_template_destroy``
> +    and ``rte_flow_actions_template_destroy``.
> +
>   * **Updated AF_XDP PMD**
>   
>     * Added support for libxdp >=v1.2.2.
> diff --git a/lib/ethdev/rte_flow.c b/lib/ethdev/rte_flow.c
> index 66614ae29b..b53f8c9b89 100644
> --- a/lib/ethdev/rte_flow.c
> +++ b/lib/ethdev/rte_flow.c
> @@ -1431,3 +1431,150 @@ rte_flow_configure(uint16_t port_id,
>   				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>   				  NULL, rte_strerror(ENOTSUP));
>   }
> +
> +struct rte_flow_pattern_template *
> +rte_flow_pattern_template_create(uint16_t port_id,
> +		const struct rte_flow_pattern_template_attr *template_attr,
> +		const struct rte_flow_item pattern[],
> +		struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +	struct rte_flow_pattern_template *template;
> +
> +	if (unlikely(!ops))
> +		return NULL;
> +	if (likely(!!ops->pattern_template_create)) {

Don't we need any state checks?

Check pattern vs NULL?

> +		template = ops->pattern_template_create(dev, template_attr,
> +						     pattern, error);
> +		if (template == NULL)
> +			flow_err(port_id, -rte_errno, error);
> +		return template;
> +	}
> +	rte_flow_error_set(error, ENOTSUP,
> +			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +			   NULL, rte_strerror(ENOTSUP));
> +	return NULL;
> +}
> +
> +int
> +rte_flow_pattern_template_destroy(uint16_t port_id,
> +		struct rte_flow_pattern_template *pattern_template,
> +		struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +
> +	if (unlikely(!ops))
> +		return -rte_errno;
> +	if (likely(!!ops->pattern_template_destroy)) {

IMHO we should return success here if pattern_template is NULL

> +		return flow_err(port_id,
> +				ops->pattern_template_destroy(dev,
> +							      pattern_template,
> +							      error),
> +				error);
> +	}
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				  NULL, rte_strerror(ENOTSUP));
> +}
> +
> +struct rte_flow_actions_template *
> +rte_flow_actions_template_create(uint16_t port_id,
> +			const struct rte_flow_actions_template_attr *template_attr,
> +			const struct rte_flow_action actions[],
> +			const struct rte_flow_action masks[],
> +			struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +	struct rte_flow_actions_template *template;
> +
> +	if (unlikely(!ops))
> +		return NULL;
> +	if (likely(!!ops->actions_template_create)) {

State checks?

Check actions and masks vs NULL?

> +		template = ops->actions_template_create(dev, template_attr,
> +							actions, masks, error);
> +		if (template == NULL)
> +			flow_err(port_id, -rte_errno, error);
> +		return template;
> +	}
> +	rte_flow_error_set(error, ENOTSUP,
> +			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +			   NULL, rte_strerror(ENOTSUP));
> +	return NULL;
> +}
> +
> +int
> +rte_flow_actions_template_destroy(uint16_t port_id,
> +			struct rte_flow_actions_template *actions_template,
> +			struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +
> +	if (unlikely(!ops))
> +		return -rte_errno;
> +	if (likely(!!ops->actions_template_destroy)) {

IMHO we should return success here if actions_template is NULL


> +		return flow_err(port_id,
> +				ops->actions_template_destroy(dev,
> +							      actions_template,
> +							      error),
> +				error);
> +	}
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				  NULL, rte_strerror(ENOTSUP));
> +}
> +
> +struct rte_flow_template_table *
> +rte_flow_template_table_create(uint16_t port_id,
> +			const struct rte_flow_template_table_attr *table_attr,
> +			struct rte_flow_pattern_template *pattern_templates[],
> +			uint8_t nb_pattern_templates,
> +			struct rte_flow_actions_template *actions_templates[],
> +			uint8_t nb_actions_templates,
> +			struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +	struct rte_flow_template_table *table;
> +
> +	if (unlikely(!ops))
> +		return NULL;
> +	if (likely(!!ops->template_table_create)) {

Argument sanity checks here. array NULL when size is not 0.

> +		table = ops->template_table_create(dev, table_attr,
> +					pattern_templates, nb_pattern_templates,
> +					actions_templates, nb_actions_templates,
> +					error);
> +		if (table == NULL)
> +			flow_err(port_id, -rte_errno, error);
> +		return table;
> +	}
> +	rte_flow_error_set(error, ENOTSUP,
> +			   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +			   NULL, rte_strerror(ENOTSUP));
> +	return NULL;
> +}
> +
> +int
> +rte_flow_template_table_destroy(uint16_t port_id,
> +				struct rte_flow_template_table *template_table,
> +				struct rte_flow_error *error)
> +{
> +	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
> +	const struct rte_flow_ops *ops = rte_flow_ops_get(port_id, error);
> +
> +	if (unlikely(!ops))
> +		return -rte_errno;
> +	if (likely(!!ops->template_table_destroy)) {

Return success if template_table is NULL

> +		return flow_err(port_id,
> +				ops->template_table_destroy(dev,
> +							    template_table,
> +							    error),
> +				error);
> +	}
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				  NULL, rte_strerror(ENOTSUP));
> +}
> diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
> index 92be2a9a89..e87db5a540 100644
> --- a/lib/ethdev/rte_flow.h
> +++ b/lib/ethdev/rte_flow.h
> @@ -4961,6 +4961,266 @@ rte_flow_configure(uint16_t port_id,
>   		   const struct rte_flow_port_attr *port_attr,
>   		   struct rte_flow_error *error);
>   
> +/**
> + * Opaque type returned after successful creation of pattern template.
> + * This handle can be used to manage the created pattern template.
> + */
> +struct rte_flow_pattern_template;
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Flow pattern template attributes.
> + */
> +__extension__
> +struct rte_flow_pattern_template_attr {
> +	/**
> +	 * Relaxed matching policy.
> +	 * - PMD may match only on items with mask member set and skip
> +	 * matching on protocol layers specified without any masks.
> +	 * - If not set, PMD will match on protocol layers
> +	 * specified without any masks as well.
> +	 * - Packet data must be stacked in the same order as the
> +	 * protocol layers to match inside packets, starting from the lowest.
> +	 */
> +	uint32_t relaxed_matching:1;
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Create pattern template.

Create flow pattern template.

> + *
> + * The pattern template defines common matching fields without values.
> + * For example, matching on 5 tuple TCP flow, the template will be
> + * eth(null) + IPv4(source + dest) + TCP(s_port + d_port),
> + * while values for each rule will be set during the flow rule creation.
> + * The number and order of items in the template must be the same
> + * at the rule creation.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] template_attr
> + *   Pattern template attributes.
> + * @param[in] pattern
> + *   Pattern specification (list terminated by the END pattern item).
> + *   The spec member of an item is not used unless the end member is used.

Interpretation of the pattern may depend on transfer vs non-transfer
rule to be used. It is essential information and we should provide it
when pattern template is created.

The information is provided on table stage, but it is too late.

> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   Handle on success, NULL otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +struct rte_flow_pattern_template *
> +rte_flow_pattern_template_create(uint16_t port_id,
> +		const struct rte_flow_pattern_template_attr *template_attr,
> +		const struct rte_flow_item pattern[],
> +		struct rte_flow_error *error);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Destroy pattern template.

Destroy flow pattern template.

> + *
> + * This function may be called only when
> + * there are no more tables referencing this template.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] pattern_template
> + *   Handle of the template to be destroyed.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +int
> +rte_flow_pattern_template_destroy(uint16_t port_id,
> +		struct rte_flow_pattern_template *pattern_template,
> +		struct rte_flow_error *error);
> +
> +/**
> + * Opaque type returned after successful creation of actions template.
> + * This handle can be used to manage the created actions template.
> + */
> +struct rte_flow_actions_template;
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Flow actions template attributes.
> + */
> +struct rte_flow_actions_template_attr;
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Create actions template.

Create flow rule actions template.

> + *
> + * The actions template holds a list of action types without values.
> + * For example, the template to change TCP ports is TCP(s_port + d_port),
> + * while values for each rule will be set during the flow rule creation.
> + * The number and order of actions in the template must be the same
> + * at the rule creation.

Again, it highly depends on transfer vs non-transfer. Moreover,
application definitely know it. So, it should say if the action
is intended for transfer or non-transfer flow rule.

> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] template_attr
> + *   Template attributes.
> + * @param[in] actions
> + *   Associated actions (list terminated by the END action).
> + *   The spec member is only used if @p masks spec is non-zero.
> + * @param[in] masks
> + *   List of actions that marks which of the action's member is constant.
> + *   A mask has the same format as the corresponding action.
> + *   If the action field in @p masks is not 0,
> + *   the corresponding value in an action from @p actions will be the part
> + *   of the template and used in all flow rules.
> + *   The order of actions in @p masks is the same as in @p actions.
> + *   In case of indirect actions present in @p actions,
> + *   the actual action type should be present in @p mask.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   Handle on success, NULL otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +struct rte_flow_actions_template *
> +rte_flow_actions_template_create(uint16_t port_id,
> +		const struct rte_flow_actions_template_attr *template_attr,
> +		const struct rte_flow_action actions[],
> +		const struct rte_flow_action masks[],
> +		struct rte_flow_error *error);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Destroy actions template.

Destroy flow rule actions template.

> + *
> + * This function may be called only when
> + * there are no more tables referencing this template.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] actions_template
> + *   Handle to the template to be destroyed.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +int
> +rte_flow_actions_template_destroy(uint16_t port_id,
> +		struct rte_flow_actions_template *actions_template,
> +		struct rte_flow_error *error);
> +
> +/**
> + * Opaque type returned after successful creation of a template table.
> + * This handle can be used to manage the created template table.
> + */
> +struct rte_flow_template_table;
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Table attributes.
> + */
> +struct rte_flow_template_table_attr {
> +	/**
> +	 * Flow attributes to be used in each rule generated from this table.
> +	 */
> +	struct rte_flow_attr flow_attr;
> +	/**
> +	 * Maximum number of flow rules that this table holds.
> +	 */
> +	uint32_t nb_flows;
> +};
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Create template table.
> + *
> + * A template table consists of multiple pattern templates and actions
> + * templates associated with a single set of rule attributes (group ID,
> + * priority and traffic direction).
> + *
> + * Each rule is free to use any combination of pattern and actions templates
> + * and specify particular values for items and actions it would like to change.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] table_attr
> + *   Template table attributes.
> + * @param[in] pattern_templates
> + *   Array of pattern templates to be used in this table.
> + * @param[in] nb_pattern_templates
> + *   The number of pattern templates in the pattern_templates array.
> + * @param[in] actions_templates
> + *   Array of actions templates to be used in this table.
> + * @param[in] nb_actions_templates
> + *   The number of actions templates in the actions_templates array.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   Handle on success, NULL otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +struct rte_flow_template_table *
> +rte_flow_template_table_create(uint16_t port_id,
> +		const struct rte_flow_template_table_attr *table_attr,
> +		struct rte_flow_pattern_template *pattern_templates[],
> +		uint8_t nb_pattern_templates,
> +		struct rte_flow_actions_template *actions_templates[],
> +		uint8_t nb_actions_templates,
> +		struct rte_flow_error *error);
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice.
> + *
> + * Destroy template table.
> + *
> + * This function may be called only when
> + * there are no more flow rules referencing this table.
> + *
> + * @param port_id
> + *   Port identifier of Ethernet device.
> + * @param[in] template_table
> + *   Handle to the table to be destroyed.
> + * @param[out] error
> + *   Perform verbose error reporting if not NULL.
> + *   PMDs initialize this structure in case of error only.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +__rte_experimental
> +int
> +rte_flow_template_table_destroy(uint16_t port_id,
> +		struct rte_flow_template_table *template_table,
> +		struct rte_flow_error *error);
> +
>   #ifdef __cplusplus
>   }
>   #endif
> diff --git a/lib/ethdev/rte_flow_driver.h b/lib/ethdev/rte_flow_driver.h
> index 7c29930d0f..2d96db1dc7 100644
> --- a/lib/ethdev/rte_flow_driver.h
> +++ b/lib/ethdev/rte_flow_driver.h
> @@ -162,6 +162,43 @@ struct rte_flow_ops {
>   		(struct rte_eth_dev *dev,
>   		 const struct rte_flow_port_attr *port_attr,
>   		 struct rte_flow_error *err);
> +	/** See rte_flow_pattern_template_create() */
> +	struct rte_flow_pattern_template *(*pattern_template_create)
> +		(struct rte_eth_dev *dev,
> +		 const struct rte_flow_pattern_template_attr *template_attr,
> +		 const struct rte_flow_item pattern[],
> +		 struct rte_flow_error *err);
> +	/** See rte_flow_pattern_template_destroy() */
> +	int (*pattern_template_destroy)
> +		(struct rte_eth_dev *dev,
> +		 struct rte_flow_pattern_template *pattern_template,
> +		 struct rte_flow_error *err);
> +	/** See rte_flow_actions_template_create() */
> +	struct rte_flow_actions_template *(*actions_template_create)
> +		(struct rte_eth_dev *dev,
> +		 const struct rte_flow_actions_template_attr *template_attr,
> +		 const struct rte_flow_action actions[],
> +		 const struct rte_flow_action masks[],
> +		 struct rte_flow_error *err);
> +	/** See rte_flow_actions_template_destroy() */
> +	int (*actions_template_destroy)
> +		(struct rte_eth_dev *dev,
> +		 struct rte_flow_actions_template *actions_template,
> +		 struct rte_flow_error *err);
> +	/** See rte_flow_template_table_create() */
> +	struct rte_flow_template_table *(*template_table_create)
> +		(struct rte_eth_dev *dev,
> +		 const struct rte_flow_template_table_attr *table_attr,
> +		 struct rte_flow_pattern_template *pattern_templates[],
> +		 uint8_t nb_pattern_templates,
> +		 struct rte_flow_actions_template *actions_templates[],
> +		 uint8_t nb_actions_templates,
> +		 struct rte_flow_error *err);
> +	/** See rte_flow_template_table_destroy() */
> +	int (*template_table_destroy)
> +		(struct rte_eth_dev *dev,
> +		 struct rte_flow_template_table *template_table,
> +		 struct rte_flow_error *err);
>   };
>   
>   /**
> diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map
> index f1235aa913..5fd2108895 100644
> --- a/lib/ethdev/version.map
> +++ b/lib/ethdev/version.map
> @@ -262,6 +262,12 @@ EXPERIMENTAL {
>   	rte_eth_dev_priority_flow_ctrl_queue_info_get;
>   	rte_flow_info_get;
>   	rte_flow_configure;
> +	rte_flow_pattern_template_create;
> +	rte_flow_pattern_template_destroy;
> +	rte_flow_actions_template_create;
> +	rte_flow_actions_template_destroy;
> +	rte_flow_template_table_create;
> +	rte_flow_template_table_destroy;
>   };
>   
>   INTERNAL {