Re: [dpdk-dev] [PATCH v2] examples/ipsec-secgw: enable rte_flow based packet distribution

[Ankur Dwivedi <adwivedi@marvell.com>]

Hi Akhil,

Can you please review this patch?

Thanks
Ankur

>-----Original Message-----
>From: Ankur Dwivedi <adwivedi@marvell.com>
>Sent: Wednesday, June 10, 2020 2:39 PM
>To: dev@dpdk.org
>Cc: praveen.shetty@intel.com; konstantin.ananyev@intel.com;
>radu.nicolau@intel.com; akhil.goyal@nxp.com; Anoob Joseph
><anoobj@marvell.com>; Ankur Dwivedi <adwivedi@marvell.com>
>Subject: [PATCH v2] examples/ipsec-secgw: enable rte_flow based packet
>distribution
>
>From: Anoob Joseph <anoobj@marvell.com>
>
>RTE_FLOW API allows hardware parsing and steering of packets to specific
>queues which helps in distributing ingress traffic across various cores.
>Adding 'flow' rules allows user to specify the distribution required.
>
>Signed-off-by: Anoob Joseph <anoobj@marvell.com>
>---
>v2:
>* Removed Change-Id
>
> doc/guides/sample_app_ug/ipsec_secgw.rst |  78 ++++++++-
> examples/ipsec-secgw/Makefile            |   1 +
> examples/ipsec-secgw/flow.c              | 285
>+++++++++++++++++++++++++++++++
> examples/ipsec-secgw/flow.h              |  15 ++
> examples/ipsec-secgw/ipsec-secgw.c       |   3 +
> examples/ipsec-secgw/ipsec-secgw.h       |   7 +
> examples/ipsec-secgw/ipsec.h             |   7 -
> examples/ipsec-secgw/meson.build         |  15 +-
> examples/ipsec-secgw/parser.c            |  46 +++++
> examples/ipsec-secgw/parser.h            |   7 +-
> 10 files changed, 450 insertions(+), 14 deletions(-)  create mode 100644
>examples/ipsec-secgw/flow.c  create mode 100644 examples/ipsec-
>secgw/flow.h
>
>diff --git a/doc/guides/sample_app_ug/ipsec_secgw.rst
>b/doc/guides/sample_app_ug/ipsec_secgw.rst
>index 81c5d43..434f484 100644
>--- a/doc/guides/sample_app_ug/ipsec_secgw.rst
>+++ b/doc/guides/sample_app_ug/ipsec_secgw.rst
>@@ -348,7 +348,7 @@ Configurations
> --------------
>
> The following sections provide the syntax of configurations to initialize -your
>SP, SA, Routing and Neighbour tables.
>+your SP, SA, Routing, Flow and Neighbour tables.
> Configurations shall be specified in the configuration file to be passed to  the
>application. The file is then parsed by the application. The successful  parsing
>will result in the appropriate rules being applied to the tables @@ -369,7
>+369,7 @@ General rule syntax
>
> The parse treats one line in the configuration file as one configuration  item
>(unless the line concatenation symbol exists). Every configuration -item shall
>follow the syntax of either SP, SA, Routing or Neighbour
>+item shall follow the syntax of either SP, SA, Routing, Flow or
>+Neighbour
> rules specified below.
>
> The configuration parser supports the following special symbols:
>@@ -808,6 +808,80 @@ Example SP rules:
>
>     rt ipv6 dst 1111:1111:1111:1111:1111:1111:1111:5555/116 port 0
>
>+Flow rule syntax
>+^^^^^^^^^^^^^^^^
>+
>+Flow rule enables the usage of hardware classification capabilities to
>+match specific ingress traffic and redirect the packets to the
>+specified queue. This feature is optional and relies on hardware ``rte_flow``
>support.
>+
>+The flow rule syntax is shown as follows:
>+
>+.. code-block:: console
>+
>+    flow <ip_ver> <src_ip> <dst_ip> <port> <queue>
>+
>+
>+where each options means:
>+
>+``<ip_ver>``
>+
>+ * IP protocol version
>+
>+ * Optional: No
>+
>+ * Available options:
>+
>+   * *ipv4*: IP protocol version 4
>+   * *ipv6*: IP protocol version 6
>+
>+``<src_ip>``
>+
>+ * The source IP address and mask
>+
>+ * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used
>+
>+ * Syntax:
>+
>+   * *src X.X.X.X/Y* for IPv4
>+   * *src XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6
>+
>+``<dst_ip>``
>+
>+ * The destination IP address and mask
>+
>+ * Optional: Yes, default address 0.0.0.0 and mask of 0 will be used
>+
>+ * Syntax:
>+
>+   * *dst X.X.X.X/Y* for IPv4
>+   * *dst XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX/Y* for IPv6
>+
>+``<port>``
>+
>+ * The traffic input port id
>+
>+ * Optional: yes, default input port 0 will be used
>+
>+ * Syntax: *port X*
>+
>+``<queue>``
>+
>+ * The traffic input queue id
>+
>+ * Optional: yes, default input queue 0 will be used
>+
>+ * Syntax: *queue X*
>+
>+Example flow rules:
>+
>+.. code-block:: console
>+
>+    flow ipv4 dst 172.16.1.5/32 port 0 queue 0
>+
>+    flow ipv6 dst 1111:1111:1111:1111:1111:1111:1111:5555/116 port 1
>+ queue 0
>+
>+
> Neighbour rule syntax
> ^^^^^^^^^^^^^^^^^^^^^
>
>diff --git a/examples/ipsec-secgw/Makefile b/examples/ipsec-secgw/Makefile
>index c4a272a..dbae152 100644
>--- a/examples/ipsec-secgw/Makefile
>+++ b/examples/ipsec-secgw/Makefile
>@@ -18,6 +18,7 @@ SRCS-y += ipsec_process.c  SRCS-y += ipsec-secgw.c
>SRCS-y += ipsec_worker.c  SRCS-y += event_helper.c
>+SRCS-y += flow.c
>
> CFLAGS += -gdwarf-2
>
>diff --git a/examples/ipsec-secgw/flow.c b/examples/ipsec-secgw/flow.c new
>file mode 100644 index 0000000..69f8405
>--- /dev/null
>+++ b/examples/ipsec-secgw/flow.c
>@@ -0,0 +1,285 @@
>+/* SPDX-License-Identifier: BSD-3-Clause
>+ * Copyright (C) 2020 Marvell International Ltd.
>+ */
>+
>+#include <stdio.h>
>+
>+#include <rte_common.h>
>+#include <rte_flow.h>
>+#include <rte_ip.h>
>+
>+#include "flow.h"
>+#include "ipsec-secgw.h"
>+#include "parser.h"
>+
>+#define FLOW_RULES_MAX 128
>+
>+struct flow_rule_entry {
>+	uint8_t is_ipv4;
>+	RTE_STD_C11
>+	union {
>+		struct {
>+			struct rte_flow_item_ipv4 spec;
>+			struct rte_flow_item_ipv4 mask;
>+		} ipv4;
>+		struct {
>+			struct rte_flow_item_ipv6 spec;
>+			struct rte_flow_item_ipv6 mask;
>+		} ipv6;
>+	};
>+	uint16_t port;
>+	uint16_t queue;
>+	struct rte_flow *flow;
>+} flow_rule_tbl[FLOW_RULES_MAX];
>+
>+int nb_flow_rule;
>+
>+static void
>+ipv4_hdr_print(struct rte_ipv4_hdr *hdr) {
>+	char a, b, c, d;
>+
>+	uint32_t_to_char(rte_bswap32(hdr->src_addr), &a, &b, &c, &d);
>+	printf("src: %3hhu.%3hhu.%3hhu.%3hhu \t", a, b, c, d);
>+
>+	uint32_t_to_char(rte_bswap32(hdr->dst_addr), &a, &b, &c, &d);
>+	printf("dst: %3hhu.%3hhu.%3hhu.%3hhu", a, b, c, d); }
>+
>+static int
>+ipv4_addr_cpy(rte_be32_t *spec, rte_be32_t *mask, char *token,
>+	      struct parse_status *status)
>+{
>+	struct in_addr ip;
>+	uint32_t depth;
>+
>+	APP_CHECK(parse_ipv4_addr(token, &ip, &depth) == 0, status,
>+		 "unrecognized input \"%s\", expect valid ipv4 addr", token);
>+	if (status->status < 0)
>+		return -1;
>+
>+	if (depth > 32)
>+		return -1;
>+
>+	memcpy(mask, &rte_flow_item_ipv4_mask.hdr.src_addr, sizeof(ip));
>+
>+	*spec = ip.s_addr;
>+	if (depth < 32)
>+		*mask = *mask << (32-depth);
>+
>+	return 0;
>+}
>+
>+static void
>+ipv6_hdr_print(struct rte_ipv6_hdr *hdr) {
>+	uint8_t *addr;
>+
>+	addr = hdr->src_addr;
>+	printf("src: %4hx:%4hx:%4hx:%4hx:%4hx:%4hx:%4hx:%4hx \t",
>+	       (uint16_t)((addr[0] << 8) | addr[1]),
>+	       (uint16_t)((addr[2] << 8) | addr[3]),
>+	       (uint16_t)((addr[4] << 8) | addr[5]),
>+	       (uint16_t)((addr[6] << 8) | addr[7]),
>+	       (uint16_t)((addr[8] << 8) | addr[9]),
>+	       (uint16_t)((addr[10] << 8) | addr[11]),
>+	       (uint16_t)((addr[12] << 8) | addr[13]),
>+	       (uint16_t)((addr[14] << 8) | addr[15]));
>+
>+	addr = hdr->dst_addr;
>+	printf("dst: %4hx:%4hx:%4hx:%4hx:%4hx:%4hx:%4hx:%4hx",
>+	       (uint16_t)((addr[0] << 8) | addr[1]),
>+	       (uint16_t)((addr[2] << 8) | addr[3]),
>+	       (uint16_t)((addr[4] << 8) | addr[5]),
>+	       (uint16_t)((addr[6] << 8) | addr[7]),
>+	       (uint16_t)((addr[8] << 8) | addr[9]),
>+	       (uint16_t)((addr[10] << 8) | addr[11]),
>+	       (uint16_t)((addr[12] << 8) | addr[13]),
>+	       (uint16_t)((addr[14] << 8) | addr[15])); }
>+
>+static int
>+ipv6_addr_cpy(uint8_t *spec, uint8_t *mask, char *token,
>+	      struct parse_status *status)
>+{
>+	struct in6_addr ip;
>+	uint32_t depth, i;
>+
>+	APP_CHECK(parse_ipv6_addr(token, &ip, &depth) == 0, status,
>+		"unrecognized input \"%s\", expect valid ipv6 address",
>token);
>+	if (status->status < 0)
>+		return -1;
>+
>+	memcpy(mask, &rte_flow_item_ipv6_mask.hdr.src_addr, sizeof(ip));
>+	memcpy(spec, ip.s6_addr, sizeof(struct in6_addr));
>+
>+	for (i = 0; i < depth && (i%8 <= sizeof(struct in6_addr)); i++)
>+		mask[i/8] &= ~(1 << (7-i%8));
>+
>+	return 0;
>+}
>+
>+void
>+parse_flow_tokens(char **tokens, uint32_t n_tokens,
>+		  struct parse_status *status)
>+{
>+	struct flow_rule_entry *rule;
>+	uint32_t ti;
>+
>+	if (nb_flow_rule >= FLOW_RULES_MAX) {
>+		printf("Too many flow rules\n");
>+		return;
>+	}
>+
>+	rule = &flow_rule_tbl[nb_flow_rule];
>+	memset(rule, 0, sizeof(*rule));
>+
>+	if (strcmp(tokens[0], "ipv4") == 0) {
>+		rule->is_ipv4 = 1;
>+	} else if (strcmp(tokens[0], "ipv6") == 0) {
>+		rule->is_ipv4 = 0;
>+	} else {
>+		APP_CHECK(0, status, "unrecognized input \"%s\"", tokens[0]);
>+		return;
>+	}
>+
>+	for (ti = 1; ti < n_tokens; ti++) {
>+		if (strcmp(tokens[ti], "src") == 0) {
>+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
>+			if (status->status < 0)
>+				return;
>+
>+			if (rule->is_ipv4) {
>+				if (ipv4_addr_cpy(&rule-
>>ipv4.spec.hdr.src_addr,
>+						  &rule-
>>ipv4.mask.hdr.src_addr,
>+						  tokens[ti], status))
>+					return;
>+			} else {
>+				if (ipv6_addr_cpy(rule-
>>ipv6.spec.hdr.src_addr,
>+						  rule-
>>ipv6.mask.hdr.src_addr,
>+						  tokens[ti], status))
>+					return;
>+			}
>+		}
>+		if (strcmp(tokens[ti], "dst") == 0) {
>+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
>+			if (status->status < 0)
>+				return;
>+
>+			if (rule->is_ipv4) {
>+				if (ipv4_addr_cpy(&rule-
>>ipv4.spec.hdr.dst_addr,
>+						  &rule-
>>ipv4.mask.hdr.dst_addr,
>+						  tokens[ti], status))
>+					return;
>+			} else {
>+				if (ipv6_addr_cpy(rule-
>>ipv6.spec.hdr.dst_addr,
>+						  rule-
>>ipv6.mask.hdr.dst_addr,
>+						  tokens[ti], status))
>+					return;
>+			}
>+		}
>+
>+		if (strcmp(tokens[ti], "port") == 0) {
>+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
>+			if (status->status < 0)
>+				return;
>+			APP_CHECK_TOKEN_IS_NUM(tokens, ti, status);
>+			if (status->status < 0)
>+				return;
>+
>+			rule->port = atoi(tokens[ti]);
>+
>+			INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
>+			if (status->status < 0)
>+				return;
>+			APP_CHECK_TOKEN_IS_NUM(tokens, ti, status);
>+			if (status->status < 0)
>+				return;
>+
>+			rule->queue = atoi(tokens[ti]);
>+		}
>+	}
>+
>+	nb_flow_rule++;
>+}
>+
>+#define MAX_RTE_FLOW_PATTERN (3)
>+#define MAX_RTE_FLOW_ACTIONS (2)
>+
>+static void
>+flow_init_single(struct flow_rule_entry *rule) {
>+	struct rte_flow_item pattern[MAX_RTE_FLOW_PATTERN] = {};
>+	struct rte_flow_action action[MAX_RTE_FLOW_ACTIONS] = {};
>+	struct rte_flow_attr attr = {};
>+	struct rte_flow_error err;
>+	int ret;
>+
>+	attr.egress = 0;
>+	attr.ingress = 1;
>+
>+	action[0].type = RTE_FLOW_ACTION_TYPE_QUEUE;
>+	action[0].conf = &(struct rte_flow_action_queue) {
>+				.index = rule->queue,
>+	};
>+	action[1].type = RTE_FLOW_ACTION_TYPE_END;
>+
>+	pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
>+
>+	if (rule->is_ipv4) {
>+		pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
>+		pattern[1].spec = &rule->ipv4.spec;
>+		pattern[1].mask = &rule->ipv4.mask;
>+	} else {
>+		pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
>+		pattern[1].spec = &rule->ipv6.spec;
>+		pattern[1].mask = &rule->ipv6.mask;
>+	}
>+
>+	pattern[2].type = RTE_FLOW_ITEM_TYPE_END;
>+
>+	ret = rte_flow_validate(rule->port, &attr, pattern, action, &err);
>+	if (ret < 0) {
>+		RTE_LOG(ERR, IPSEC, "Flow validation failed %s\n",
>err.message);
>+		return;
>+	}
>+
>+	rule->flow = rte_flow_create(rule->port, &attr, pattern, action, &err);
>+	if (rule->flow == NULL)
>+		RTE_LOG(ERR, IPSEC, "Flow creation return %s\n",
>err.message); }
>+
>+void
>+flow_init(void)
>+{
>+	struct flow_rule_entry *rule;
>+	int i;
>+
>+	for (i = 0; i < nb_flow_rule; i++) {
>+		rule = &flow_rule_tbl[i];
>+		flow_init_single(rule);
>+	}
>+
>+	for (i = 0; i < nb_flow_rule; i++) {
>+		rule = &flow_rule_tbl[i];
>+		if (rule->is_ipv4) {
>+			printf("Flow #%3d: spec ipv4 ", i);
>+			ipv4_hdr_print(&rule->ipv4.spec.hdr);
>+			printf("\n");
>+			printf("           mask ipv4 ");
>+			ipv4_hdr_print(&rule->ipv4.mask.hdr);
>+		} else {
>+			printf("Flow #%3d: spec ipv6 ", i);
>+			ipv6_hdr_print(&rule->ipv6.spec.hdr);
>+			printf("\n");
>+			printf("           mask ipv6 ");
>+			ipv6_hdr_print(&rule->ipv6.mask.hdr);
>+		}
>+
>+		printf("\tPort: %d, Queue: %d", rule->port, rule->queue);
>+
>+		if (rule->flow == NULL)
>+			printf(" [UNSUPPORTED]");
>+		printf("\n");
>+	}
>+}
>diff --git a/examples/ipsec-secgw/flow.h b/examples/ipsec-secgw/flow.h new
>file mode 100644 index 0000000..1b1b477
>--- /dev/null
>+++ b/examples/ipsec-secgw/flow.h
>@@ -0,0 +1,15 @@
>+/* SPDX-License-Identifier: BSD-3-Clause
>+ * Copyright (C) 2020 Marvell International Ltd.
>+ */
>+
>+#ifndef _FLOW_H_
>+#define _FLOW_H_
>+
>+#include "parser.h"
>+
>+void parse_flow_tokens(char **tokens, uint32_t n_tokens,
>+		       struct parse_status *status);
>+
>+void flow_init(void);
>+
>+#endif /* _FLOW_H_ */
>diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-
>secgw/ipsec-secgw.c
>index f777ce2..d19688d 100644
>--- a/examples/ipsec-secgw/ipsec-secgw.c
>+++ b/examples/ipsec-secgw/ipsec-secgw.c
>@@ -49,6 +49,7 @@
> #include <rte_ip_frag.h>
>
> #include "event_helper.h"
>+#include "flow.h"
> #include "ipsec.h"
> #include "ipsec_worker.h"
> #include "parser.h"
>@@ -2914,6 +2915,8 @@ struct lcore_conf {
> 		}
> 	}
>
>+	flow_init();
>+
> 	check_all_ports_link_status(enabled_port_mask);
>
> 	/* launch per-lcore init on every lcore */ diff --git a/examples/ipsec-
>secgw/ipsec-secgw.h b/examples/ipsec-secgw/ipsec-secgw.h
>index 4b53cb5..412d727 100644
>--- a/examples/ipsec-secgw/ipsec-secgw.h
>+++ b/examples/ipsec-secgw/ipsec-secgw.h
>@@ -34,6 +34,13 @@
> 	((uint64_t)(a) & 0xff))
> #endif
>
>+#define uint32_t_to_char(ip, a, b, c, d) do {\
>+		*a = (uint8_t)(ip >> 24 & 0xff);\
>+		*b = (uint8_t)(ip >> 16 & 0xff);\
>+		*c = (uint8_t)(ip >> 8 & 0xff);\
>+		*d = (uint8_t)(ip & 0xff);\
>+	} while (0)
>+
> #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0))
>
> struct traffic_type {
>diff --git a/examples/ipsec-secgw/ipsec.h b/examples/ipsec-secgw/ipsec.h
>index 2f69199..7031e28 100644
>--- a/examples/ipsec-secgw/ipsec.h
>+++ b/examples/ipsec-secgw/ipsec.h
>@@ -28,13 +28,6 @@
> #define IV_OFFSET		(sizeof(struct rte_crypto_op) + \
> 				sizeof(struct rte_crypto_sym_op))
>
>-#define uint32_t_to_char(ip, a, b, c, d) do {\
>-		*a = (uint8_t)(ip >> 24 & 0xff);\
>-		*b = (uint8_t)(ip >> 16 & 0xff);\
>-		*c = (uint8_t)(ip >> 8 & 0xff);\
>-		*d = (uint8_t)(ip & 0xff);\
>-	} while (0)
>-
> #define DEFAULT_MAX_CATEGORIES	1
>
> #define INVALID_SPI (0)
>diff --git a/examples/ipsec-secgw/meson.build b/examples/ipsec-
>secgw/meson.build
>index f9ba2a2..d0373da 100644
>--- a/examples/ipsec-secgw/meson.build
>+++ b/examples/ipsec-secgw/meson.build
>@@ -9,6 +9,17 @@
> deps += ['security', 'lpm', 'acl', 'hash', 'ip_frag', 'ipsec', 'eventdev']
>allow_experimental_apis = true  sources = files(
>-	'esp.c', 'event_helper.c', 'ipsec.c', 'ipsec_process.c', 'ipsec-secgw.c',
>-	'ipsec_worker.c', 'parser.c', 'rt.c', 'sa.c', 'sad.c', 'sp4.c', 'sp6.c'
>+	'esp.c',
>+	'event_helper.c',
>+	'flow.c',
>+	'ipsec.c',
>+	'ipsec_process.c',
>+	'ipsec-secgw.c',
>+	'ipsec_worker.c',
>+	'parser.c',
>+	'rt.c',
>+	'sa.c',
>+	'sad.c',
>+	'sp4.c',
>+	'sp6.c'
> )
>diff --git a/examples/ipsec-secgw/parser.c b/examples/ipsec-secgw/parser.c
>index 65eb7e9..8f66660 100644
>--- a/examples/ipsec-secgw/parser.c
>+++ b/examples/ipsec-secgw/parser.c
>@@ -11,6 +11,7 @@
> #include <cmdline_socket.h>
> #include <cmdline.h>
>
>+#include "flow.h"
> #include "ipsec.h"
> #include "parser.h"
>
>@@ -484,6 +485,49 @@ struct cfg_rt_add_cfg_item {
> 	},
> };
>
>+/* flow add parse */
>+struct cfg_flow_add_cfg_item {
>+	cmdline_fixed_string_t flow_keyword;
>+	cmdline_multi_string_t multi_string;
>+};
>+
>+static void
>+cfg_flow_add_cfg_item_parsed(void *parsed_result,
>+	__rte_unused struct cmdline *cl, void *data) {
>+	struct cfg_flow_add_cfg_item *params = parsed_result;
>+	char *tokens[32];
>+	uint32_t n_tokens = RTE_DIM(tokens);
>+	struct parse_status *status = (struct parse_status *)data;
>+
>+	APP_CHECK(parse_tokenize_string(
>+		params->multi_string, tokens, &n_tokens) == 0,
>+		status, "too many arguments\n");
>+	if (status->status < 0)
>+		return;
>+
>+	parse_flow_tokens(tokens, n_tokens, status); }
>+
>+static cmdline_parse_token_string_t cfg_flow_add_flow_str =
>+	TOKEN_STRING_INITIALIZER(struct cfg_flow_add_cfg_item,
>+		flow_keyword, "flow");
>+
>+static cmdline_parse_token_string_t cfg_flow_add_multi_str =
>+	TOKEN_STRING_INITIALIZER(struct cfg_flow_add_cfg_item,
>multi_string,
>+		TOKEN_STRING_MULTI);
>+
>+cmdline_parse_inst_t cfg_flow_add_rule = {
>+	.f = cfg_flow_add_cfg_item_parsed,
>+	.data = NULL,
>+	.help_str = "",
>+	.tokens = {
>+		(void *) &cfg_flow_add_flow_str,
>+		(void *) &cfg_flow_add_multi_str,
>+		NULL,
>+	},
>+};
>+
> /* neigh add parse */
> struct cfg_neigh_add_item {
> 	cmdline_fixed_string_t neigh;
>@@ -538,6 +582,7 @@ struct cfg_neigh_add_item {
> 	(cmdline_parse_inst_t *)&cfg_sp_add_rule,
> 	(cmdline_parse_inst_t *)&cfg_sa_add_rule,
> 	(cmdline_parse_inst_t *)&cfg_rt_add_rule,
>+	(cmdline_parse_inst_t *)&cfg_flow_add_rule,
> 	(cmdline_parse_inst_t *)&cfg_neigh_add_rule,
> 	NULL,
> };
>@@ -564,6 +609,7 @@ struct cfg_neigh_add_item {
> 	cfg_sp_add_rule.data = &status;
> 	cfg_sa_add_rule.data = &status;
> 	cfg_rt_add_rule.data = &status;
>+	cfg_flow_add_rule.data = &status;
> 	cfg_neigh_add_rule.data = &status;
>
> 	do {
>diff --git a/examples/ipsec-secgw/parser.h b/examples/ipsec-secgw/parser.h
>index 6e764fe..a0ff7e1 100644
>--- a/examples/ipsec-secgw/parser.h
>+++ b/examples/ipsec-secgw/parser.h
>@@ -2,12 +2,13 @@
>  * Copyright(c) 2016 Intel Corporation
>  */
>
>+#ifndef __PARSER_H
>+#define __PARSER_H
>+
> #include <sys/types.h>
> #include <netinet/in.h>
> #include <netinet/ip.h>
>-
>-#ifndef __PARSER_H
>-#define __PARSER_H
>+#include <string.h>
>
> struct parse_status {
> 	int status;
>--
>1.9.3