[dpdk-dev] [RFC] malloc: add malloc and free log function

[dpdk-dev] [RFC] malloc: add malloc and free log function

[Xueming Li]

DPDK uses second level memory allocation management, this makes
regular memory profiler tool not applicant. This patch trys to
provide a lightweight malloc and free logging, then show leaked
memory entries based on logs.

This tool only target to malloc and free tracking, for memzone
used by ring and mempool, "dump_memzone" in testpmd list them.

There will be another example that enable and dump tracking as
secondary process.

This tool came from Mellanox internal Hackathon, thanks Shahaf Shuler
<shahafs@mellanox.com> who provided the idea.


Xueming Li (1):
  malloc: add malloc and free log function

 app/test-pmd/cmdline.c                      |  61 ++++++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 +++
 lib/librte_eal/common/eal_memcfg.h          |  18 ++++
 lib/librte_eal/common/include/rte_malloc.h  |  30 +++++-
 lib/librte_eal/common/malloc_elem.h         |   4 +-
 lib/librte_eal/common/rte_malloc.c          | 154 +++++++++++++++++++++++++++-
 lib/librte_eal/rte_eal_version.map          |   2 +
 7 files changed, 280 insertions(+), 4 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [RFC] malloc: add malloc and free log function

[Xueming Li]

This patch introduces new feature to track rte_malloc leakage by logging
malloc and free function.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/cmdline.c                      |  61 ++++++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 +++
 lib/librte_eal/common/eal_memcfg.h          |  18 ++++
 lib/librte_eal/common/include/rte_malloc.h  |  30 +++++-
 lib/librte_eal/common/malloc_elem.h         |   4 +-
 lib/librte_eal/common/rte_malloc.c          | 154 +++++++++++++++++++++++++++-
 lib/librte_eal/rte_eal_version.map          |   2 +
 7 files changed, 280 insertions(+), 4 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index a037a55..274e391 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -9554,6 +9554,8 @@ struct cmd_rm_mirror_rule_result {
 
 struct cmd_dump_result {
 	cmdline_fixed_string_t dump;
+	cmdline_fixed_string_t cmd;
+	uint32_t val;
 };
 
 static void
@@ -9576,6 +9578,16 @@ static void cmd_dump_parsed(void *parsed_result,
 		rte_dump_physmem_layout(stdout);
 	else if (!strcmp(res->dump, "dump_memzone"))
 		rte_memzone_dump(stdout);
+	else if (!strcmp(res->dump, "dump_malloc")) {
+		if (!strcmp(res->cmd, "start")) {
+			if (rte_malloc_log_init(res->val))
+				fprintf(stdout, "Failed to start logging\n");
+		} else if (!strcmp(res->cmd, "dump")) {
+			rte_malloc_log_dump(stdout, res->val);
+		} else if (!strcmp(res->cmd, "stop")) {
+			rte_malloc_log_init(0);
+		}
+	}
 	else if (!strcmp(res->dump, "dump_struct_sizes"))
 		dump_struct_sizes();
 	else if (!strcmp(res->dump, "dump_ring"))
@@ -9595,9 +9607,53 @@ static void cmd_dump_parsed(void *parsed_result,
 		"dump_struct_sizes#"
 		"dump_ring#"
 		"dump_mempool#"
+		"dump_malloc#"
 		"dump_devargs#"
 		"dump_log_types");
-
+cmdline_parse_token_string_t cmd_dump_malloc =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump,
+		"dump_malloc");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_start =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"start");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_dump =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"dump");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_stop =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"stop");
+cmdline_parse_token_num_t cmd_dump_val =
+	TOKEN_NUM_INITIALIZER(struct cmd_dump_result, val, UINT32);
+
+cmdline_parse_inst_t cmd_dump_malloc_start = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Start rte_malloc tracking log with <count> buffer entries",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_start,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_dump = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Dump rte_malloc tracking log with <level>, 0:summary, 1:leaks, 2: all",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_dump,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_stop = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Stop rte_malloc tracking log",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_stop,
+		NULL,
+	},
+};
 cmdline_parse_inst_t cmd_dump = {
 	.f = cmd_dump_parsed,  /* function to call */
 	.data = NULL,      /* 2nd arg of func */
@@ -19452,6 +19508,9 @@ struct cmd_showport_macs_result {
 	(cmdline_parse_inst_t *)&cmd_showport_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_config_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_start,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_stop,
 	(cmdline_parse_inst_t *)&cmd_dump_one,
 	(cmdline_parse_inst_t *)&cmd_ethertype_filter,
 	(cmdline_parse_inst_t *)&cmd_syn_filter,
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index 5bb12a5..1a9879f 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -561,6 +561,21 @@ Dumps the statistics of all or specific memory pool::
 
    testpmd> dump_mempool [mempool_name]
 
+dump malloc
+~~~~~~~~~~~~
+
+Start rte_malloc and rte_free tracking with number of buffers::
+
+   testpmd> dump_malloc start <count>
+
+Dump tracking result with different level, 0:summary, 1:leaks, 2: all::
+
+   testpmd> dump_malloc dump <level>
+
+Stop tracking::
+
+   testpmd> dump_malloc stop
+
 dump devargs
 ~~~~~~~~~~~~
 
diff --git a/lib/librte_eal/common/eal_memcfg.h b/lib/librte_eal/common/eal_memcfg.h
index 583fcb5..38055b4 100644
--- a/lib/librte_eal/common/eal_memcfg.h
+++ b/lib/librte_eal/common/eal_memcfg.h
@@ -15,6 +15,20 @@
 #include "malloc_heap.h"
 
 /**
+ * Memory allocation and free tracking log
+ */
+struct rte_malloc_log {
+	int socket;
+	void *addr;
+	size_t size; /* requested size */
+	size_t real_size;
+	size_t align;
+	int8_t free; /* allocate or free. */
+	int8_t alloc_free; /* alloc and free both tracked. */
+	char name[64]; /* name may come from stack, copy. */
+};
+
+/**
  * Memory configuration shared across multiple processes.
  */
 struct rte_mem_config {
@@ -73,6 +87,10 @@ struct rte_mem_config {
 	/**< TSC rate */
 
 	uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
+
+	struct rte_malloc_log *malloc_logs; /**< Log entries. */
+	int malloc_log_max; /**< Max number of logs to write. */
+	int malloc_log_index; /**< Next log index to write to. */
 };
 
 /* update internal config from shared mem config */
diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h
index 42ca051..fb116fd 100644
--- a/lib/librte_eal/common/include/rte_malloc.h
+++ b/lib/librte_eal/common/include/rte_malloc.h
@@ -508,7 +508,35 @@ struct rte_malloc_socket_stats {
  *   to dump all objects.
  */
 void
-rte_malloc_dump_stats(FILE *f, const char *type);
+rte_malloc_dump_stats(FILE *out, const char *type);
+
+/**
+ * Initialize malloc tracking log buffer.
+ *
+ * @param count
+ *   Max count of tracking log entries, 0 to stop tracking
+ * @return
+ *   0 on success, negative errno otherwise
+ */
+__rte_experimental
+int
+rte_malloc_log_init(int count);
+
+
+/**
+ * Dump malloc tracking log to output.
+ *
+ * @param out
+ *   output file handle
+ * @param detail
+ *   detail level of output
+ *   0: summary
+ *   1: potential leaks - allocated without free
+ *  -1: all entries
+ */
+__rte_experimental
+void
+rte_malloc_log_dump(FILE *out, int detail);
 
 /**
  * Dump contents of all malloc heaps to a file.
diff --git a/lib/librte_eal/common/malloc_elem.h b/lib/librte_eal/common/malloc_elem.h
index a1e5f7f..6c6dba8 100644
--- a/lib/librte_eal/common/malloc_elem.h
+++ b/lib/librte_eal/common/malloc_elem.h
@@ -27,7 +27,9 @@ struct malloc_elem {
 	LIST_ENTRY(malloc_elem) free_list;
 	/**< list of free elements in heap */
 	struct rte_memseg_list *msl;
-	volatile enum elem_state state;
+	volatile uint32_t state:2;
+	volatile uint32_t log:1;		/* Element logged. */
+	volatile uint32_t log_index:29;		/* Element log index. */
 	uint32_t pad;
 	size_t size;
 	struct malloc_elem *orig_elem;
diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
index d6026a2..9926518 100644
--- a/lib/librte_eal/common/rte_malloc.c
+++ b/lib/librte_eal/common/rte_malloc.c
@@ -6,6 +6,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include <sys/queue.h>
 
 #include <rte_errno.h>
@@ -29,10 +30,154 @@
 #include "eal_private.h"
 
 
+/*
+ * Track and log memory allocation information.
+ */
+static void
+rte_malloc_log(const char *type, size_t size, unsigned int align,
+	       void *addr)
+{
+	struct rte_malloc_log *log;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !elem || !mcfg->malloc_log_max ||
+	    mcfg->malloc_log_max == mcfg->malloc_log_index)
+		return;
+
+	log = &mcfg->malloc_logs[mcfg->malloc_log_index];
+	if (type)
+		strncpy(log->name, type, sizeof(log->name) - 1);
+	log->size = size;
+	log->real_size = elem->size;
+	align = align ? align : 1;
+	log->align = RTE_CACHE_LINE_ROUNDUP(align);
+	log->addr = addr;
+	log->socket = elem->heap->socket_id;
+	elem->log = 1;
+	elem->log_index = mcfg->malloc_log_index++;
+}
+
+/*
+ * track and log memory free
+ */
+static void
+rte_free_log(void *addr)
+{
+	struct rte_malloc_log *alloc_log;
+	struct rte_malloc_log *log;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !elem || !mcfg->malloc_log_max ||
+	    mcfg->malloc_log_max == mcfg->malloc_log_index)
+		return;
+	log = &mcfg->malloc_logs[mcfg->malloc_log_index++];
+	log->addr = addr;
+	log->free = 1;
+	log->real_size = elem->size;
+	if (elem->log) {
+		alloc_log = &mcfg->malloc_logs[elem->log_index];
+		strncpy(log->name, alloc_log->name, sizeof(log->name));
+		log->size = alloc_log->size;
+		log->align = alloc_log->align;
+		log->socket = alloc_log->socket;
+		log->alloc_free = 1;
+		alloc_log->alloc_free = 1;
+	}
+}
+
+/*
+ * Initialize malloc tracking with max count of log entries
+ */
+int
+rte_malloc_log_init(int count)
+{
+	struct malloc_elem *elem;
+	int i;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct rte_malloc_log *log;
+
+	if (mcfg->malloc_log_max) {
+		/* release all logs. */
+		for (i = 0; i < mcfg->malloc_log_index; i++) {
+			log = &mcfg->malloc_logs[i];
+			/* clear log info from malloc elem. */
+			if (log->free || log->alloc_free)
+				continue;
+			elem = malloc_elem_from_data(log->addr);
+			if (elem) {
+				elem->log = 0;
+				elem->log_index = 0;
+			}
+		}
+		rte_free(mcfg->malloc_logs);
+		mcfg->malloc_logs = NULL;
+		mcfg->malloc_log_max = 0;
+		mcfg->malloc_log_index = 0;
+	}
+	if (count) {
+		/* allocate logs. */
+		mcfg->malloc_logs = rte_calloc("malloc_logs", count,
+					       sizeof(*log), 0);
+		if (!mcfg->malloc_logs)
+			return -ENOMEM;
+		mcfg->malloc_log_max = count;
+	}
+	return 0;
+}
+
+/*
+ * Dump malloc tracking
+ */
+void
+rte_malloc_log_dump(FILE *out, int detail)
+{
+	struct rte_malloc_log *log;
+	int i;
+	int n_alloc_free = 0;
+	int n_alloc = 0;
+	int n_free_alloc = 0;
+	int n_free = 0;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!mcfg->malloc_log_max || !mcfg->malloc_logs) {
+		fprintf(out, "warning: malloc tracking log not enabled\n");
+		return;
+	}
+	if (detail)
+		fprintf(out, "Socket Action Paired            Address    Size Align Mgmt   Total Type\n");
+	for (i = 0; i < mcfg->malloc_log_index; i++) {
+		log = &mcfg->malloc_logs[i];
+		if (detail == 2 ||
+		    (detail == 1 && !log->alloc_free && !log->free))
+			fprintf(out, "%6d %6s %6s %18p %7lu %5lu %4d %7lu %s\n",
+				log->socket,
+				log->free ? "free" : "alloc",
+				log->alloc_free ? "y" : "n",
+				log->addr, log->size, log->align,
+				MALLOC_ELEM_OVERHEAD,
+				log->real_size,
+				log->name);
+		if (log->free) {
+			n_free++;
+			if (log->alloc_free)
+				n_free_alloc++;
+		} else {
+			n_alloc++;
+			if (log->alloc_free)
+				n_alloc_free++;
+		}
+	}
+	fprintf(out, "Total rte_malloc: %d/%d, rte_free:%d/%d\n",
+		n_alloc_free, n_alloc, n_free_alloc, n_free);
+}
+
 /* Free the memory space back to heap */
 void rte_free(void *addr)
 {
 	if (addr == NULL) return;
+	rte_free_log(addr);
 	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
 		RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
 }
@@ -44,6 +189,8 @@ void rte_free(void *addr)
 rte_malloc_socket(const char *type, size_t size, unsigned int align,
 		int socket_arg)
 {
+	void *ret;
+
 	/* return NULL if size is 0 or alignment is not power-of-2 */
 	if (size == 0 || (align && !rte_is_power_of_2(align)))
 		return NULL;
@@ -57,8 +204,13 @@ void rte_free(void *addr)
 				!rte_eal_has_hugepages())
 		socket_arg = SOCKET_ID_ANY;
 
-	return malloc_heap_alloc(type, size, socket_arg, 0,
+	ret = malloc_heap_alloc(type, size, socket_arg, 0,
 			align == 0 ? 1 : align, 0, false);
+
+	if (ret)
+		rte_malloc_log(type, size, align == 0 ? 1 : align, ret);
+
+	return ret;
 }
 
 /*
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index f9ede5b..3db892d 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -338,4 +338,6 @@ EXPERIMENTAL {
 
 	# added in 20.05
 	rte_log_can_log;
+    rte_malloc_log_init;
+    rte_malloc_log_dump;
 };
-- 
1.8.3.1

Re: [dpdk-dev] [RFC] malloc: add malloc and free log function

[Burakov, Anatoly]

On 03-Apr-20 8:54 AM, Xueming Li wrote:
> This patch introduces new feature to track rte_malloc leakage by logging
> malloc and free function.

Hi,

Thanks for the patch.

A general comment - i would avoid mixing testpmd code with adding a new 
API to malloc. I understand this is an RFC so it's OK for now, but by 
the time V1 comes i think it is better to add malloc API as a separate 
patch.

> 
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>   app/test-pmd/cmdline.c                      |  61 ++++++++++-
>   doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 +++
>   lib/librte_eal/common/eal_memcfg.h          |  18 ++++
>   lib/librte_eal/common/include/rte_malloc.h  |  30 +++++-
>   lib/librte_eal/common/malloc_elem.h         |   4 +-
>   lib/librte_eal/common/rte_malloc.c          | 154 +++++++++++++++++++++++++++-
>   lib/librte_eal/rte_eal_version.map          |   2 +
>   7 files changed, 280 insertions(+), 4 deletions(-)
> 
> diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
> index a037a55..274e391 100644
> --- a/app/test-pmd/cmdline.c
> +++ b/app/test-pmd/cmdline.c
> @@ -9554,6 +9554,8 @@ struct cmd_rm_mirror_rule_result {
>   
>   struct cmd_dump_result {
>   	cmdline_fixed_string_t dump;
> +	cmdline_fixed_string_t cmd;
> +	uint32_t val;
>   };
>   
>   static void
> @@ -9576,6 +9578,16 @@ static void cmd_dump_parsed(void *parsed_result,
>   		rte_dump_physmem_layout(stdout);
>   	else if (!strcmp(res->dump, "dump_memzone"))
>   		rte_memzone_dump(stdout);
> +	else if (!strcmp(res->dump, "dump_malloc")) {
> +		if (!strcmp(res->cmd, "start")) {
> +			if (rte_malloc_log_init(res->val))
> +				fprintf(stdout, "Failed to start logging\n");
> +		} else if (!strcmp(res->cmd, "dump")) {
> +			rte_malloc_log_dump(stdout, res->val);
> +		} else if (!strcmp(res->cmd, "stop")) {
> +			rte_malloc_log_init(0);

This looks odd and confusing. I think an explicit "stop" API is needed 
here, even though technically you could get away with just one API.

> +		}
> +	}
>   	else if (!strcmp(res->dump, "dump_struct_sizes"))
>   		dump_struct_sizes();
>   	else if (!strcmp(res->dump, "dump_ring"))
> @@ -9595,9 +9607,53 @@ static void cmd_dump_parsed(void *parsed_result,
>   		"dump_struct_sizes#"
>   		"dump_ring#"
>   		"dump_mempool#"
> +		"dump_malloc#"
>   		"dump_devargs#"
>   		"dump_log_types");
> -
> +cmdline_parse_token_string_t cmd_dump_malloc =
> +	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump,
> +		"dump_malloc");
> +cmdline_parse_token_string_t cmd_dump_malloc_cmd_start =
> +	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
> +		"start");
> +cmdline_parse_token_string_t cmd_dump_malloc_cmd_dump =
> +	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
> +		"dump");
> +cmdline_parse_token_string_t cmd_dump_malloc_cmd_stop =
> +	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
> +		"stop");
> +cmdline_parse_token_num_t cmd_dump_val =
> +	TOKEN_NUM_INITIALIZER(struct cmd_dump_result, val, UINT32);
> +
> +cmdline_parse_inst_t cmd_dump_malloc_start = {
> +	.f = cmd_dump_parsed,  /* function to call */
> +	.help_str = "Start rte_malloc tracking log with <count> buffer entries",
> +	.tokens = {        /* token list, NULL terminated */
> +		(void *)&cmd_dump_malloc,
> +		(void *)&cmd_dump_malloc_cmd_start,
> +		(void *)&cmd_dump_val,
> +		NULL,
> +	},
> +};
> +cmdline_parse_inst_t cmd_dump_malloc_dump = {
> +	.f = cmd_dump_parsed,  /* function to call */
> +	.help_str = "Dump rte_malloc tracking log with <level>, 0:summary, 1:leaks, 2: all",
> +	.tokens = {        /* token list, NULL terminated */
> +		(void *)&cmd_dump_malloc,
> +		(void *)&cmd_dump_malloc_cmd_dump,
> +		(void *)&cmd_dump_val,
> +		NULL,
> +	},
> +};
> +cmdline_parse_inst_t cmd_dump_malloc_stop = {
> +	.f = cmd_dump_parsed,  /* function to call */
> +	.help_str = "Stop rte_malloc tracking log",
> +	.tokens = {        /* token list, NULL terminated */
> +		(void *)&cmd_dump_malloc,
> +		(void *)&cmd_dump_malloc_cmd_stop,
> +		NULL,
> +	},
> +};
>   cmdline_parse_inst_t cmd_dump = {
>   	.f = cmd_dump_parsed,  /* function to call */
>   	.data = NULL,      /* 2nd arg of func */
> @@ -19452,6 +19508,9 @@ struct cmd_showport_macs_result {
>   	(cmdline_parse_inst_t *)&cmd_showport_rss_hash_key,
>   	(cmdline_parse_inst_t *)&cmd_config_rss_hash_key,
>   	(cmdline_parse_inst_t *)&cmd_dump,
> +	(cmdline_parse_inst_t *)&cmd_dump_malloc_start,
> +	(cmdline_parse_inst_t *)&cmd_dump_malloc_dump,
> +	(cmdline_parse_inst_t *)&cmd_dump_malloc_stop,
>   	(cmdline_parse_inst_t *)&cmd_dump_one,
>   	(cmdline_parse_inst_t *)&cmd_ethertype_filter,
>   	(cmdline_parse_inst_t *)&cmd_syn_filter,
> diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> index 5bb12a5..1a9879f 100644
> --- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> +++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
> @@ -561,6 +561,21 @@ Dumps the statistics of all or specific memory pool::
>   
>      testpmd> dump_mempool [mempool_name]
>   
> +dump malloc
> +~~~~~~~~~~~~
> +
> +Start rte_malloc and rte_free tracking with number of buffers::
> +
> +   testpmd> dump_malloc start <count>
> +
> +Dump tracking result with different level, 0:summary, 1:leaks, 2: all::
> +
> +   testpmd> dump_malloc dump <level>
> +
> +Stop tracking::
> +
> +   testpmd> dump_malloc stop
> +
>   dump devargs
>   ~~~~~~~~~~~~
>   
> diff --git a/lib/librte_eal/common/eal_memcfg.h b/lib/librte_eal/common/eal_memcfg.h
> index 583fcb5..38055b4 100644
> --- a/lib/librte_eal/common/eal_memcfg.h
> +++ b/lib/librte_eal/common/eal_memcfg.h
> @@ -15,6 +15,20 @@
>   #include "malloc_heap.h"
>   
>   /**
> + * Memory allocation and free tracking log
> + */
> +struct rte_malloc_log {
> +	int socket;
> +	void *addr;
> +	size_t size; /* requested size */
> +	size_t real_size;

I'm not sure of the terminology here. If you're referring to outer 
element size and pad, then you should really use the same terms here as 
in the other malloc code. No need to introduce new ones.

> +	size_t align;
> +	int8_t free; /* allocate or free. */
> +	int8_t alloc_free; /* alloc and free both tracked. */

Why not just store a flag? E.g.

#define RTE_MALLOC_LOG_ALLOC (1 << 0)
#define RTE_MALLOC_LOG_FREE (1 << 1)
log->flags |= RTE_MALLOC_LOG_ALLOC;

Also, i don't see you handling rte_realloc() anywhere. While it could 
cause deallocation/new allocation, it can also resize existing allocated 
memory.

> +	char name[64]; /* name may come from stack, copy. */
> +};
> +
> +/**
>    * Memory configuration shared across multiple processes.
>    */
>   struct rte_mem_config {
> @@ -73,6 +87,10 @@ struct rte_mem_config {
>   	/**< TSC rate */
>   
>   	uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
> +
> +	struct rte_malloc_log *malloc_logs; /**< Log entries. */
> +	int malloc_log_max; /**< Max number of logs to write. */
> +	int malloc_log_index; /**< Next log index to write to. */

I'm not 100% sure of this approach. I understand why you're doing it, 
but i have this nagging thought that there probably is a better way :D

>   };
>   
>   /* update internal config from shared mem config */
> diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h
> index 42ca051..fb116fd 100644
> --- a/lib/librte_eal/common/include/rte_malloc.h
> +++ b/lib/librte_eal/common/include/rte_malloc.h
> @@ -508,7 +508,35 @@ struct rte_malloc_socket_stats {
>    *   to dump all objects.
>    */
>   void
> -rte_malloc_dump_stats(FILE *f, const char *type);
> +rte_malloc_dump_stats(FILE *out, const char *type);
> +
> +/**
> + * Initialize malloc tracking log buffer.
> + *
> + * @param count
> + *   Max count of tracking log entries, 0 to stop tracking
> + * @return
> + *   0 on success, negative errno otherwise
> + */
> +__rte_experimental
> +int
> +rte_malloc_log_init(int count);
> +
> +
> +/**
> + * Dump malloc tracking log to output.
> + *
> + * @param out
> + *   output file handle
> + * @param detail
> + *   detail level of output
> + *   0: summary
> + *   1: potential leaks - allocated without free
> + *  -1: all entries
> + */
> +__rte_experimental
> +void
> +rte_malloc_log_dump(FILE *out, int detail);
>   
>   /**
>    * Dump contents of all malloc heaps to a file.
> diff --git a/lib/librte_eal/common/malloc_elem.h b/lib/librte_eal/common/malloc_elem.h
> index a1e5f7f..6c6dba8 100644
> --- a/lib/librte_eal/common/malloc_elem.h
> +++ b/lib/librte_eal/common/malloc_elem.h
> @@ -27,7 +27,9 @@ struct malloc_elem {
>   	LIST_ENTRY(malloc_elem) free_list;
>   	/**< list of free elements in heap */
>   	struct rte_memseg_list *msl;
> -	volatile enum elem_state state;
> +	volatile uint32_t state:2;
> +	volatile uint32_t log:1;		/* Element logged. */
> +	volatile uint32_t log_index:29;		/* Element log index. */
>   	uint32_t pad;
>   	size_t size;
>   	struct malloc_elem *orig_elem;
> diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
> index d6026a2..9926518 100644
> --- a/lib/librte_eal/common/rte_malloc.c
> +++ b/lib/librte_eal/common/rte_malloc.c
> @@ -6,6 +6,7 @@
>   #include <stddef.h>
>   #include <stdio.h>
>   #include <string.h>
> +#include <assert.h>
>   #include <sys/queue.h>
>   
>   #include <rte_errno.h>
> @@ -29,10 +30,154 @@
>   #include "eal_private.h"
>   
>   
> +/*
> + * Track and log memory allocation information.
> + */
> +static void
> +rte_malloc_log(const char *type, size_t size, unsigned int align,
> +	       void *addr)
> +{
> +	struct rte_malloc_log *log;
> +	struct malloc_elem *elem = malloc_elem_from_data(addr);
> +	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> +
> +	if (!addr || !elem || !mcfg->malloc_log_max ||
> +	    mcfg->malloc_log_max == mcfg->malloc_log_index)
> +		return;
> +
> +	log = &mcfg->malloc_logs[mcfg->malloc_log_index];
> +	if (type)
> +		strncpy(log->name, type, sizeof(log->name) - 1);
> +	log->size = size;
> +	log->real_size = elem->size;
> +	align = align ? align : 1;
> +	log->align = RTE_CACHE_LINE_ROUNDUP(align);
> +	log->addr = addr;
> +	log->socket = elem->heap->socket_id;
> +	elem->log = 1;
> +	elem->log_index = mcfg->malloc_log_index++;
> +}
> +
> +/*
> + * track and log memory free
> + */
> +static void
> +rte_free_log(void *addr)
> +{
> +	struct rte_malloc_log *alloc_log;
> +	struct rte_malloc_log *log;
> +	struct malloc_elem *elem = malloc_elem_from_data(addr);
> +	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> +
> +	if (!addr || !elem || !mcfg->malloc_log_max ||
> +	    mcfg->malloc_log_max == mcfg->malloc_log_index)
> +		return;

This would still succeed when addr is invalid, because you don't do all 
of the checks that malloc_heap_free() does (you don't chec malloc 
cookies and ELEM_BUSY).

> +	log = &mcfg->malloc_logs[mcfg->malloc_log_index++];
> +	log->addr = addr;
> +	log->free = 1;
> +	log->real_size = elem->size;
> +	if (elem->log) {
> +		alloc_log = &mcfg->malloc_logs[elem->log_index];
> +		strncpy(log->name, alloc_log->name, sizeof(log->name));
> +		log->size = alloc_log->size;
> +		log->align = alloc_log->align;
> +		log->socket = alloc_log->socket;
> +		log->alloc_free = 1;
> +		alloc_log->alloc_free = 1;
> +	}
> +}
> +
> +/*
> + * Initialize malloc tracking with max count of log entries
> + */
> +int
> +rte_malloc_log_init(int count)
> +{
> +	struct malloc_elem *elem;
> +	int i;
> +	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> +	struct rte_malloc_log *log;

Since your malloc elements can only hold 29 bit indices, you should 
check the count against that value.

> +
> +	if (mcfg->malloc_log_max) {
> +		/* release all logs. */
> +		for (i = 0; i < mcfg->malloc_log_index; i++) {
> +			log = &mcfg->malloc_logs[i];
> +			/* clear log info from malloc elem. */
> +			if (log->free || log->alloc_free)
> +				continue;
> +			elem = malloc_elem_from_data(log->addr);
> +			if (elem) {
> +				elem->log = 0;
> +				elem->log_index = 0;
> +			}
> +		}
> +		rte_free(mcfg->malloc_logs);
> +		mcfg->malloc_logs = NULL;
> +		mcfg->malloc_log_max = 0;
> +		mcfg->malloc_log_index = 0;
> +	}

I don't think immediately deallocating everything on a log_init call is 
a good idea. That's why you need stop() API :)

> +	if (count) {
> +		/* allocate logs. */
> +		mcfg->malloc_logs = rte_calloc("malloc_logs", count,
> +					       sizeof(*log), 0);
> +		if (!mcfg->malloc_logs)
> +			return -ENOMEM;
> +		mcfg->malloc_log_max = count;
> +	}
> +	return 0;
> +}
> +
> +/*
> + * Dump malloc tracking
> + */
> +void
> +rte_malloc_log_dump(FILE *out, int detail)
> +{
> +	struct rte_malloc_log *log;
> +	int i;
> +	int n_alloc_free = 0;
> +	int n_alloc = 0;
> +	int n_free_alloc = 0;
> +	int n_free = 0;
> +	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
> +
> +	if (!mcfg->malloc_log_max || !mcfg->malloc_logs) {
> +		fprintf(out, "warning: malloc tracking log not enabled\n");
> +		return;
> +	}
> +	if (detail)
> +		fprintf(out, "Socket Action Paired            Address    Size Align Mgmt   Total Type\n");
> +	for (i = 0; i < mcfg->malloc_log_index; i++) {
> +		log = &mcfg->malloc_logs[i];
> +		if (detail == 2 ||
> +		    (detail == 1 && !log->alloc_free && !log->free))
> +			fprintf(out, "%6d %6s %6s %18p %7lu %5lu %4d %7lu %s\n",
> +				log->socket,
> +				log->free ? "free" : "alloc",
> +				log->alloc_free ? "y" : "n",
> +				log->addr, log->size, log->align,
> +				MALLOC_ELEM_OVERHEAD,
> +				log->real_size,
> +				log->name);
> +		if (log->free) {
> +			n_free++;
> +			if (log->alloc_free)
> +				n_free_alloc++;
> +		} else {
> +			n_alloc++;
> +			if (log->alloc_free)
> +				n_alloc_free++;
> +		}
> +	}
> +	fprintf(out, "Total rte_malloc: %d/%d, rte_free:%d/%d\n",
> +		n_alloc_free, n_alloc, n_free_alloc, n_free);
> +}
> +
>   /* Free the memory space back to heap */
>   void rte_free(void *addr)
>   {
>   	if (addr == NULL) return;
> +	rte_free_log(addr);
>   	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
>   		RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
>   }
> @@ -44,6 +189,8 @@ void rte_free(void *addr)
>   rte_malloc_socket(const char *type, size_t size, unsigned int align,
>   		int socket_arg)
>   {
> +	void *ret;
> +
>   	/* return NULL if size is 0 or alignment is not power-of-2 */
>   	if (size == 0 || (align && !rte_is_power_of_2(align)))
>   		return NULL;
> @@ -57,8 +204,13 @@ void rte_free(void *addr)
>   				!rte_eal_has_hugepages())
>   		socket_arg = SOCKET_ID_ANY;
>   
> -	return malloc_heap_alloc(type, size, socket_arg, 0,
> +	ret = malloc_heap_alloc(type, size, socket_arg, 0,
>   			align == 0 ? 1 : align, 0, false);
> +
> +	if (ret)
> +		rte_malloc_log(type, size, align == 0 ? 1 : align, ret);
> +
> +	return ret;
>   }
>   
>   /*
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index f9ede5b..3db892d 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -338,4 +338,6 @@ EXPERIMENTAL {
>   
>   	# added in 20.05
>   	rte_log_can_log;
> +    rte_malloc_log_init;
> +    rte_malloc_log_dump;
>   };
> 


-- 
Thanks,
Anatoly

[dpdk-dev] [PATCH v1 0/2] malloc: support malloc and free tracking log

[Xueming Li]

DPDK uses second level memory allocation management, this makes
regular memory profiler tool not applicant. This patch trys to
provide a lightweight malloc and free logging, then show leaked
memory entries based on logs.

This tool only target to malloc and free tracking, for memzone
used by ring and mempool, "dump_memzone" in testpmd list them.

There will be another example that enable and dump tracking as
secondary process.

This tool came from Mellanox internal Hackathon, thanks Shahaf Shuler
<shahafs@mellanox.com> who provided the idea.

V0: initial version
V1: log rte_realloc function
    rename log size to req_size, real_size to size(total)
    log padding and include padding in total size
    rename log "free" field to "type", alloc_free to "paired"
    rename malloc_log_index to malloc_log_count
    split testpmd part into separate patch
    add rte_malloc_log_stop() function
    add log entry number check
    change log dump detail level from (-1,0,1) to (0,1,2)


Xueming Li (2):
  malloc: support malloc and free tracking log
  app/testpmd: support malloc and free tracking log

 app/test-pmd/cmdline.c                      |  60 ++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++
 lib/librte_eal/common/eal_memcfg.h          |  26 +++
 lib/librte_eal/common/malloc_elem.h         |   6 +-
 lib/librte_eal/common/rte_malloc.c          | 259 +++++++++++++++++++++++++++-
 lib/librte_eal/include/rte_malloc.h         |  39 ++++-
 lib/librte_eal/rte_eal_version.map          |   3 +
 7 files changed, 399 insertions(+), 9 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [PATCH v1 1/2] malloc: support malloc and free tracking log

[Xueming Li]

This patch introduces new feature to track rte_malloc leakage by logging
malloc and free.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 lib/librte_eal/common/eal_memcfg.h  |  26 ++++
 lib/librte_eal/common/malloc_elem.h |   6 +-
 lib/librte_eal/common/rte_malloc.c  | 259 +++++++++++++++++++++++++++++++++++-
 lib/librte_eal/include/rte_malloc.h |  39 +++++-
 lib/librte_eal/rte_eal_version.map  |   3 +
 5 files changed, 325 insertions(+), 8 deletions(-)

diff --git a/lib/librte_eal/common/eal_memcfg.h b/lib/librte_eal/common/eal_memcfg.h
index 583fcb5..49d59d8 100644
--- a/lib/librte_eal/common/eal_memcfg.h
+++ b/lib/librte_eal/common/eal_memcfg.h
@@ -14,6 +14,28 @@
 
 #include "malloc_heap.h"
 
+
+enum rte_malloc_log_type {
+	rte_malloc_log_malloc,
+	rte_malloc_log_realloc,
+	rte_malloc_log_free,
+};
+
+/**
+ * Memory allocation and free tracking log
+ */
+struct rte_malloc_log {
+	int socket;
+	void *addr;
+	size_t req_size; /* requested size */
+	size_t size; /* outer element and pad */
+	size_t align;
+	uint32_t pad;
+	uint32_t type:2; /* log entry type. */
+	uint32_t paired:1; /* allocation and free both tracked. */
+	char name[64]; /* name may come from stack, copy. */
+};
+
 /**
  * Memory configuration shared across multiple processes.
  */
@@ -73,6 +95,10 @@ struct rte_mem_config {
 	/**< TSC rate */
 
 	uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
+
+	struct rte_malloc_log *malloc_logs; /**< Log entries. */
+	int malloc_log_max; /**< Max number of logs to write. */
+	int malloc_log_count; /**< count of logs written. */
 };
 
 /* update internal config from shared mem config */
diff --git a/lib/librte_eal/common/malloc_elem.h b/lib/librte_eal/common/malloc_elem.h
index a1e5f7f..006480c 100644
--- a/lib/librte_eal/common/malloc_elem.h
+++ b/lib/librte_eal/common/malloc_elem.h
@@ -9,6 +9,8 @@
 
 #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
 
+#define MALLOC_TRACKING_MAX (1 << 29)
+
 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
 struct malloc_heap;
 
@@ -27,7 +29,9 @@ struct malloc_elem {
 	LIST_ENTRY(malloc_elem) free_list;
 	/**< list of free elements in heap */
 	struct rte_memseg_list *msl;
-	volatile enum elem_state state;
+	volatile uint32_t state:2;
+	volatile uint32_t log:1;		/* Element logged. */
+	volatile uint32_t log_index:29;		/* Element log index. */
 	uint32_t pad;
 	size_t size;
 	struct malloc_elem *orig_elem;
diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
index d6026a2..046b50d 100644
--- a/lib/librte_eal/common/rte_malloc.c
+++ b/lib/librte_eal/common/rte_malloc.c
@@ -6,6 +6,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include <sys/queue.h>
 
 #include <rte_errno.h>
@@ -29,12 +30,243 @@
 #include "eal_private.h"
 
 
+const char *rte_malloc_log_type_name[] = { "malloc", "realloc", "free" };
+
+/*
+ * Track and log memory allocation information.
+ */
+static void
+rte_malloc_log(const char *type, size_t size, unsigned int align,
+	       void *addr)
+{
+	struct rte_malloc_log *log;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !elem || !mcfg->malloc_log_max ||
+	    mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count];
+	if (type)
+		strncpy(log->name, type, sizeof(log->name) - 1);
+	log->req_size = size;
+	log->pad = elem->pad;
+	log->size = elem->size + elem->pad;
+	align = align ? align : 1;
+	log->align = RTE_CACHE_LINE_ROUNDUP(align);
+	log->addr = addr;
+	log->socket = elem->heap->socket_id;
+	elem->log = 1;
+	elem->log_index = mcfg->malloc_log_count++;
+}
+
+/*
+ * track and log memory realloc
+ */
+static void
+rte_realloc_log(uint32_t pref_malloc, uint32_t prev_index,
+		void *new_addr, size_t size, unsigned int align)
+{
+	struct rte_malloc_log *prev_log;
+	struct rte_malloc_log *log;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct malloc_elem *elem = malloc_elem_from_data(new_addr);
+
+	if (!mcfg->malloc_log_max || !elem)
+		return;
+	if (pref_malloc) {
+		prev_log = &mcfg->malloc_logs[prev_index];
+		prev_log->paired = 1;
+	}
+	if (mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count++];
+	log->addr = new_addr;
+	log->type = rte_malloc_log_realloc;
+	log->req_size = size;
+	if (pref_malloc)
+		strncpy(log->name, prev_log->name, sizeof(log->name));
+	log->pad = elem->pad;
+	log->size = elem->size + elem->pad;
+	align = align ? align : 1;
+	log->align = RTE_CACHE_LINE_ROUNDUP(align);
+	log->socket = elem->heap->socket_id;
+	elem->log = 1;
+	elem->log_index = mcfg->malloc_log_count++;
+}
+
+/*
+ * track and log memory free
+ */
+static void
+rte_free_log(void *addr, uint32_t log_malloc, uint32_t log_index, size_t size)
+{
+	struct rte_malloc_log *alloc_log;
+	struct rte_malloc_log *log;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !mcfg->malloc_log_max)
+		return;
+	if (log_malloc) {
+		alloc_log = &mcfg->malloc_logs[log_index];
+		alloc_log->paired = 1;
+	}
+	if (mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count++];
+	log->addr = addr;
+	log->type = rte_malloc_log_free;
+	log->size = size;
+	if (log_malloc) {
+		strncpy(log->name, alloc_log->name, sizeof(log->name));
+		log->req_size = alloc_log->req_size;
+		log->align = alloc_log->align;
+		log->socket = alloc_log->socket;
+		log->pad = alloc_log->pad;
+		log->paired = 1;
+	}
+}
+
+/*
+ * Initialize malloc tracking with max count of log entries
+ */
+int
+rte_malloc_log_init(uint32_t count)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (mcfg->malloc_log_max) {
+		RTE_LOG(ERR, EAL, "malloc tracking started\n");
+		return -EINVAL;
+	}
+	if (!count || count > MALLOC_TRACKING_MAX) {
+		RTE_LOG(ERR, EAL, "invalid malloc tracking log number\n");
+		return -EINVAL;
+	}
+
+	/* allocate logs. */
+	mcfg->malloc_logs = rte_calloc("malloc_logs", count,
+				       sizeof(*mcfg->malloc_logs), 0);
+	if (!mcfg->malloc_logs)
+		return -ENOMEM;
+
+	mcfg->malloc_log_count = 0;
+	mcfg->malloc_log_max = count;
+	return 0;
+}
+
+/*
+ * Initialize malloc tracking with max count of log entries
+ */
+int
+rte_malloc_log_stop(void)
+{
+	struct malloc_elem *elem;
+	int i;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct rte_malloc_log *log;
+
+	if (!mcfg->malloc_log_max) {
+		RTE_LOG(ERR, EAL, "malloc tracking not started\n");
+		return -EINVAL;
+	}
+
+	/* release all logs. */
+	for (i = 0; i < mcfg->malloc_log_count; i++) {
+		log = &mcfg->malloc_logs[i];
+		/* clear log info from malloc elem. */
+		if (log->type)
+			continue;
+		elem = malloc_elem_from_data(log->addr);
+		if (elem) {
+			elem->log = 0;
+			elem->log_index = 0;
+		}
+	}
+
+	rte_free(mcfg->malloc_logs);
+	mcfg->malloc_logs = NULL;
+	mcfg->malloc_log_max = 0;
+	mcfg->malloc_log_count = 0;
+
+	return 0;
+}
+
+/*
+ * Dump malloc tracking
+ */
+void
+rte_malloc_log_dump(FILE *out, uint32_t detail)
+{
+	struct rte_malloc_log *log;
+	int i;
+	int n_alloc_free = 0;
+	int n_alloc = 0;
+	int n_free_alloc = 0;
+	int n_free = 0;
+	size_t alloc_leak = 0;
+	size_t free_leak = 0;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!mcfg->malloc_log_max || !mcfg->malloc_logs) {
+		RTE_LOG(ERR, EAL, "malloc tracking not started\n");
+		return;
+	}
+	if (detail)
+		fprintf(out, "Socket  Action Paired            Address    Size Align Mgmt   Pad Total Type\n");
+	for (i = 0; i < mcfg->malloc_log_count; i++) {
+		log = &mcfg->malloc_logs[i];
+		if (detail > 1 ||
+		    (detail == 1 && !log->paired &&
+		     log->type != rte_malloc_log_free))
+			fprintf(out, "%6d %7s %6s %18p %7lu %5lu %4d %3u %7lu %s\n",
+				log->socket,
+				rte_malloc_log_type_name[log->type],
+				log->paired ? "y" : "n",
+				log->addr, log->req_size, log->align,
+				MALLOC_ELEM_OVERHEAD,
+				log->pad,
+				log->size,
+				log->name);
+		if (log->type == rte_malloc_log_free) {
+			n_free++;
+			if (log->paired)
+				n_free_alloc++;
+			else
+				free_leak += log->size;
+		} else {
+			n_alloc++;
+			if (log->paired)
+				n_alloc_free++;
+			else
+				alloc_leak += log->size;
+		}
+	}
+	fprintf(out, "Total rte_malloc and rte_realloc: %d/%d leak %zu(B), rte_free: %d/%d leak: %zu(B)\n",
+		n_alloc_free, n_alloc, alloc_leak,
+		n_free_alloc, n_free, free_leak);
+	if (mcfg->malloc_log_count == mcfg->malloc_log_max)
+		fprintf(out, "Warning: log full, please consider larger log buffer\n");
+}
+
 /* Free the memory space back to heap */
 void rte_free(void *addr)
 {
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	uint32_t log_malloc, log_index;
+	size_t size;
+
 	if (addr == NULL) return;
-	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
+	if (elem) {
+		log_malloc = elem->log;
+		log_index = elem->log_index;
+		size = elem->size + elem->pad;
+	}
+	if (malloc_heap_free(elem) < 0)
 		RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
+	else
+		rte_free_log(addr, log_malloc, log_index, size);
 }
 
 /*
@@ -44,6 +276,8 @@ void rte_free(void *addr)
 rte_malloc_socket(const char *type, size_t size, unsigned int align,
 		int socket_arg)
 {
+	void *ret;
+
 	/* return NULL if size is 0 or alignment is not power-of-2 */
 	if (size == 0 || (align && !rte_is_power_of_2(align)))
 		return NULL;
@@ -57,8 +291,13 @@ void rte_free(void *addr)
 				!rte_eal_has_hugepages())
 		socket_arg = SOCKET_ID_ANY;
 
-	return malloc_heap_alloc(type, size, socket_arg, 0,
+	ret = malloc_heap_alloc(type, size, socket_arg, 0,
 			align == 0 ? 1 : align, 0, false);
+
+	if (ret)
+		rte_malloc_log(type, size, align == 0 ? 1 : align, ret);
+
+	return ret;
 }
 
 /*
@@ -123,10 +362,12 @@ void rte_free(void *addr)
 void *
 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket)
 {
+	struct malloc_elem *elem = malloc_elem_from_data(ptr);
+	uint32_t log_malloc, log_index;
+
 	if (ptr == NULL)
 		return rte_malloc_socket(NULL, size, align, socket);
 
-	struct malloc_elem *elem = malloc_elem_from_data(ptr);
 	if (elem == NULL) {
 		RTE_LOG(ERR, EAL, "Error: memory corruption detected\n");
 		return NULL;
@@ -139,9 +380,15 @@ void rte_free(void *addr)
 	 */
 	if ((socket == SOCKET_ID_ANY ||
 	     (unsigned int)socket == elem->heap->socket_id) &&
-			RTE_PTR_ALIGN(ptr, align) == ptr &&
-			malloc_heap_resize(elem, size) == 0)
-		return ptr;
+	    RTE_PTR_ALIGN(ptr, align) == ptr) {
+		log_malloc = elem->log;
+		log_index = elem->log_index;
+		if (malloc_heap_resize(elem, size) == 0) {
+			rte_realloc_log(log_malloc, log_index, ptr, size,
+					align);
+			return ptr;
+		}
+	}
 
 	/* either requested socket id doesn't match, alignment is off
 	 * or we have no room to expand,
diff --git a/lib/librte_eal/include/rte_malloc.h b/lib/librte_eal/include/rte_malloc.h
index 42ca051..4947ba9 100644
--- a/lib/librte_eal/include/rte_malloc.h
+++ b/lib/librte_eal/include/rte_malloc.h
@@ -508,7 +508,44 @@ struct rte_malloc_socket_stats {
  *   to dump all objects.
  */
 void
-rte_malloc_dump_stats(FILE *f, const char *type);
+rte_malloc_dump_stats(FILE *out, const char *type);
+
+/**
+ * Initialize malloc tracking log buffer.
+ *
+ * @param count
+ *   Max count of tracking log entries, range (1 : 1 << 29)
+ * @return
+ *   0 on success, negative errno otherwise
+ */
+__rte_experimental
+int
+rte_malloc_log_init(uint32_t count);
+
+/**
+ * Stop malloc tracking log.
+ *
+ * @return
+ *   0 on success, negative errno otherwise
+ */
+__rte_experimental
+int
+rte_malloc_log_stop(void);
+
+/**
+ * Dump malloc tracking log to output.
+ *
+ * @param out
+ *   output file handle
+ * @param detail
+ *   detail level of output
+ *   0: summary
+ *   1: potential leaks - allocated without free
+ *   2: all entries
+ */
+__rte_experimental
+void
+rte_malloc_log_dump(FILE *out, uint32_t detail);
 
 /**
  * Dump contents of all malloc heaps to a file.
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index f9ede5b..25c4b13 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -338,4 +338,7 @@ EXPERIMENTAL {
 
 	# added in 20.05
 	rte_log_can_log;
+    rte_malloc_log_init;
+    rte_malloc_log_dump;
+    rte_malloc_log_stop;
 };
-- 
1.8.3.1

[dpdk-dev] [PATCH v1 2/2] app/testpmd: support malloc and free tracking log

[Xueming Li]

New CLI commands to manipulate malloc tracking log:

dump_malloc start <n>: start malloc tracking with number of buffers
dump_malloc dump <n>:  dump mmalloc tracking log with detail level
dump_malloc stop:      stop malloc tracking

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/cmdline.c                      | 60 ++++++++++++++++++++++++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst | 15 ++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 863b567..0490823 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -9554,6 +9554,8 @@ struct cmd_rm_mirror_rule_result {
 
 struct cmd_dump_result {
 	cmdline_fixed_string_t dump;
+	cmdline_fixed_string_t cmd;
+	uint32_t val;
 };
 
 static void
@@ -9628,6 +9630,16 @@ static void cmd_dump_parsed(void *parsed_result,
 		dump_socket_mem(stdout);
 	else if (!strcmp(res->dump, "dump_memzone"))
 		rte_memzone_dump(stdout);
+	else if (!strcmp(res->dump, "dump_malloc")) {
+		if (!strcmp(res->cmd, "start")) {
+			if (rte_malloc_log_init(res->val))
+				fprintf(stdout, "Failed to start logging\n");
+		} else if (!strcmp(res->cmd, "dump")) {
+			rte_malloc_log_dump(stdout, res->val);
+		} else if (!strcmp(res->cmd, "stop")) {
+			rte_malloc_log_stop();
+		}
+	}
 	else if (!strcmp(res->dump, "dump_struct_sizes"))
 		dump_struct_sizes();
 	else if (!strcmp(res->dump, "dump_ring"))
@@ -9650,7 +9662,50 @@ static void cmd_dump_parsed(void *parsed_result,
 		"dump_mempool#"
 		"dump_devargs#"
 		"dump_log_types");
-
+cmdline_parse_token_string_t cmd_dump_malloc =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump,
+		"dump_malloc");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_start =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"start");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_dump =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"dump");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_stop =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"stop");
+cmdline_parse_token_num_t cmd_dump_val =
+	TOKEN_NUM_INITIALIZER(struct cmd_dump_result, val, UINT32);
+
+cmdline_parse_inst_t cmd_dump_malloc_start = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Start rte_malloc tracking log with <count> buffer entries",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_start,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_dump = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Dump rte_malloc tracking log with <level>, 0:summary, 1:leaks, 2: all",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_dump,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_stop = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Stop rte_malloc tracking log",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_stop,
+		NULL,
+	},
+};
 cmdline_parse_inst_t cmd_dump = {
 	.f = cmd_dump_parsed,  /* function to call */
 	.data = NULL,      /* 2nd arg of func */
@@ -19505,6 +19560,9 @@ struct cmd_showport_macs_result {
 	(cmdline_parse_inst_t *)&cmd_showport_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_config_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_start,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_stop,
 	(cmdline_parse_inst_t *)&cmd_dump_one,
 	(cmdline_parse_inst_t *)&cmd_ethertype_filter,
 	(cmdline_parse_inst_t *)&cmd_syn_filter,
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index dcee5de..1c55f23 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -567,6 +567,21 @@ Dumps the statistics of all or specific memory pool::
 
    testpmd> dump_mempool [mempool_name]
 
+dump malloc
+~~~~~~~~~~~~
+
+Start rte_malloc and rte_free tracking with number of buffers::
+
+   testpmd> dump_malloc start <count>
+
+Dump tracking result with different level, 0:summary, 1:leaks, 2: all::
+
+   testpmd> dump_malloc dump <level>
+
+Stop tracking::
+
+   testpmd> dump_malloc stop
+
 dump devargs
 ~~~~~~~~~~~~
 
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH v1 1/2] malloc: support malloc and free tracking log

[Stephen Hemminger]

On Wed,  8 Apr 2020 04:04:53 +0000
Xueming Li <xuemingl@mellanox.com> wrote:

> 	int n_alloc_free = 0;
> +	int n_alloc = 0;
> +	int n_free_alloc = 0;
> +	int n_free = 0;
> +	size_t alloc_leak = 0;

All statistics should be unsigned, size_t or uint64_t to avoid any wraparound issues.

Re: [dpdk-dev] [PATCH v1 1/2] malloc: support malloc and free tracking log

[Stephen Hemminger]

On Wed,  8 Apr 2020 04:04:53 +0000
Xueming Li <xuemingl@mellanox.com> wrote:

>  * Dump contents of all malloc heaps to a file.
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index f9ede5b..25c4b13 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -338,4 +338,7 @@ EXPERIMENTAL {
>  
>  	# added in 20.05
>  	rte_log_can_log;
> +    rte_malloc_log_init;
> +    rte_malloc_log_dump;
> +    rte_malloc_log_stop;
>  };

Looks like other entries were indented with tabs and you used spaces?

Re: [dpdk-dev] [PATCH v1 1/2] malloc: support malloc and free tracking log

[Xueming(Steven) Li]

Thanks for your suggestion, will include them in next version.

> -----Original Message-----
> From: Stephen Hemminger <stephen@networkplumber.org>
> Sent: Wednesday, April 8, 2020 1:12 PM
> To: Xueming(Steven) Li <xuemingl@mellanox.com>
> Cc: Anatoly Burakov <anatoly.burakov@intel.com>; Ferruh Yigit
> <ferruh.yigit@intel.com>; dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> Subject: Re: [dpdk-dev] [PATCH v1 1/2] malloc: support malloc and free
> tracking log
> 
> On Wed,  8 Apr 2020 04:04:53 +0000
> Xueming Li <xuemingl@mellanox.com> wrote:
> 
> >  * Dump contents of all malloc heaps to a file.
> > diff --git a/lib/librte_eal/rte_eal_version.map
> b/lib/librte_eal/rte_eal_version.map
> > index f9ede5b..25c4b13 100644
> > --- a/lib/librte_eal/rte_eal_version.map
> > +++ b/lib/librte_eal/rte_eal_version.map
> > @@ -338,4 +338,7 @@ EXPERIMENTAL {
> >
> >  	# added in 20.05
> >  	rte_log_can_log;
> > +    rte_malloc_log_init;
> > +    rte_malloc_log_dump;
> > +    rte_malloc_log_stop;
> >  };
> 
> Looks like other entries were indented with tabs and you used spaces?

Re: [dpdk-dev] [PATCH v1 2/2] app/testpmd: support malloc and free tracking log

[Iremonger, Bernard]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Wednesday, April 8, 2020 5:05 AM
> To: Burakov, Anatoly <anatoly.burakov@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> Subject: [dpdk-dev] [PATCH v1 2/2] app/testpmd: support malloc and free
> tracking log
> 
> New CLI commands to manipulate malloc tracking log:
> 
> dump_malloc start <n>: start malloc tracking with number of buffers
> dump_malloc dump <n>:  dump mmalloc tracking log with detail level
> dump_malloc stop:      stop malloc tracking
> 
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>

Acked-by: Bernard Iremonger <bernard.iremonger@intel.com

[dpdk-dev] [RFC v2 0/2] malloc: support malloc and free tracking log

[Xueming Li]

DPDK uses second level memory allocation management, this makes
regular memory profiler tool not applicant. This patch trys to
provide a lightweight malloc and free logging, then show leaked
memory entries based on logs.

This tool only target to malloc and free tracking, for memzone
used by ring and mempool, "dump_memzone" in testpmd list them.

There will be another example that enable and dump tracking as
secondary process.

This tool came from Mellanox internal Hackathon, thanks Shahaf Shuler
<shahafs@mellanox.com> who provided the idea.

V0: initial version
V1: log rte_realloc function
    rename log size to req_size, real_size to size(total)
    log padding and include padding in total size
    rename log "free" field to "type", alloc_free to "paired"
    rename malloc_log_index to malloc_log_count
    split testpmd part into separate patch
    add rte_malloc_log_stop() function
    add log entry number check
    change log dump detail level from (-1,0,1) to (0,1,2)
V2: fix map file indent
    change type of counter variables to unsigned
    fix CI error

Xueming Li (2):
  malloc: support malloc and free tracking log
  app/testpmd: support malloc and free tracking log

 app/test-pmd/cmdline.c                      |  60 ++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst |  15 ++
 lib/librte_eal/common/eal_memcfg.h          |  26 +++
 lib/librte_eal/common/malloc_elem.h         |   6 +-
 lib/librte_eal/common/rte_malloc.c          | 261 +++++++++++++++++++++++++++-
 lib/librte_eal/include/rte_malloc.h         |  39 ++++-
 lib/librte_eal/rte_eal_version.map          |   3 +
 7 files changed, 401 insertions(+), 9 deletions(-)

-- 
1.8.3.1

[dpdk-dev] [RFC v2 1/2] malloc: support malloc and free tracking log

[Xueming Li]

This patch introduces new feature to track rte_malloc leakage by logging
malloc and free.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 lib/librte_eal/common/eal_memcfg.h  |  26 ++++
 lib/librte_eal/common/malloc_elem.h |   6 +-
 lib/librte_eal/common/rte_malloc.c  | 261 +++++++++++++++++++++++++++++++++++-
 lib/librte_eal/include/rte_malloc.h |  39 +++++-
 lib/librte_eal/rte_eal_version.map  |   3 +
 5 files changed, 327 insertions(+), 8 deletions(-)

diff --git a/lib/librte_eal/common/eal_memcfg.h b/lib/librte_eal/common/eal_memcfg.h
index 583fcb5..49d59d8 100644
--- a/lib/librte_eal/common/eal_memcfg.h
+++ b/lib/librte_eal/common/eal_memcfg.h
@@ -14,6 +14,28 @@
 
 #include "malloc_heap.h"
 
+
+enum rte_malloc_log_type {
+	rte_malloc_log_malloc,
+	rte_malloc_log_realloc,
+	rte_malloc_log_free,
+};
+
+/**
+ * Memory allocation and free tracking log
+ */
+struct rte_malloc_log {
+	int socket;
+	void *addr;
+	size_t req_size; /* requested size */
+	size_t size; /* outer element and pad */
+	size_t align;
+	uint32_t pad;
+	uint32_t type:2; /* log entry type. */
+	uint32_t paired:1; /* allocation and free both tracked. */
+	char name[64]; /* name may come from stack, copy. */
+};
+
 /**
  * Memory configuration shared across multiple processes.
  */
@@ -73,6 +95,10 @@ struct rte_mem_config {
 	/**< TSC rate */
 
 	uint8_t dma_maskbits; /**< Keeps the more restricted dma mask. */
+
+	struct rte_malloc_log *malloc_logs; /**< Log entries. */
+	int malloc_log_max; /**< Max number of logs to write. */
+	int malloc_log_count; /**< count of logs written. */
 };
 
 /* update internal config from shared mem config */
diff --git a/lib/librte_eal/common/malloc_elem.h b/lib/librte_eal/common/malloc_elem.h
index a1e5f7f..006480c 100644
--- a/lib/librte_eal/common/malloc_elem.h
+++ b/lib/librte_eal/common/malloc_elem.h
@@ -9,6 +9,8 @@
 
 #define MIN_DATA_SIZE (RTE_CACHE_LINE_SIZE)
 
+#define MALLOC_TRACKING_MAX (1 << 29)
+
 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
 struct malloc_heap;
 
@@ -27,7 +29,9 @@ struct malloc_elem {
 	LIST_ENTRY(malloc_elem) free_list;
 	/**< list of free elements in heap */
 	struct rte_memseg_list *msl;
-	volatile enum elem_state state;
+	volatile uint32_t state:2;
+	volatile uint32_t log:1;		/* Element logged. */
+	volatile uint32_t log_index:29;		/* Element log index. */
 	uint32_t pad;
 	size_t size;
 	struct malloc_elem *orig_elem;
diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
index d6026a2..2e30693 100644
--- a/lib/librte_eal/common/rte_malloc.c
+++ b/lib/librte_eal/common/rte_malloc.c
@@ -6,6 +6,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
+#include <assert.h>
 #include <sys/queue.h>
 
 #include <rte_errno.h>
@@ -29,12 +30,245 @@
 #include "eal_private.h"
 
 
+const char *rte_malloc_log_type_name[] = { "malloc", "realloc", "free" };
+
+/*
+ * Track and log memory allocation information.
+ */
+static void
+rte_malloc_log(const char *type, size_t size, unsigned int align,
+	       void *addr)
+{
+	struct rte_malloc_log *log;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !elem || !mcfg->malloc_log_max ||
+	    mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count];
+	if (type)
+		strncpy(log->name, type, sizeof(log->name) - 1);
+	log->req_size = size;
+	log->pad = elem->pad;
+	log->size = elem->size + elem->pad;
+	align = align ? align : 1;
+	log->align = RTE_CACHE_LINE_ROUNDUP(align);
+	log->addr = addr;
+	log->socket = elem->heap->socket_id;
+	elem->log = 1;
+	elem->log_index = mcfg->malloc_log_count++;
+}
+
+/*
+ * track and log memory realloc
+ */
+static void
+rte_realloc_log(uint32_t pref_malloc, uint32_t prev_index,
+		void *new_addr, size_t size, unsigned int align)
+{
+	struct rte_malloc_log *prev_log;
+	struct rte_malloc_log *log;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct malloc_elem *elem = malloc_elem_from_data(new_addr);
+
+	if (!mcfg->malloc_log_max || !elem)
+		return;
+	if (pref_malloc) {
+		prev_log = &mcfg->malloc_logs[prev_index];
+		prev_log->paired = 1;
+	}
+	if (mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count++];
+	log->addr = new_addr;
+	log->type = rte_malloc_log_realloc;
+	log->req_size = size;
+	if (pref_malloc)
+		strncpy(log->name, prev_log->name, sizeof(log->name));
+	log->pad = elem->pad;
+	log->size = elem->size + elem->pad;
+	align = align ? align : 1;
+	log->align = RTE_CACHE_LINE_ROUNDUP(align);
+	log->socket = elem->heap->socket_id;
+	elem->log = 1;
+	elem->log_index = mcfg->malloc_log_count++;
+}
+
+/*
+ * track and log memory free
+ */
+static void
+rte_free_log(void *addr, uint32_t log_malloc, uint32_t log_index, size_t size)
+{
+	struct rte_malloc_log *alloc_log;
+	struct rte_malloc_log *log;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!addr || !mcfg->malloc_log_max)
+		return;
+	if (log_malloc) {
+		alloc_log = &mcfg->malloc_logs[log_index];
+		alloc_log->paired = 1;
+	}
+	if (mcfg->malloc_log_max == mcfg->malloc_log_count)
+		return;
+	log = &mcfg->malloc_logs[mcfg->malloc_log_count++];
+	log->addr = addr;
+	log->type = rte_malloc_log_free;
+	log->size = size;
+	if (log_malloc) {
+		strncpy(log->name, alloc_log->name, sizeof(log->name));
+		log->req_size = alloc_log->req_size;
+		log->align = alloc_log->align;
+		log->socket = alloc_log->socket;
+		log->pad = alloc_log->pad;
+		log->paired = 1;
+	}
+}
+
+/*
+ * Initialize malloc tracking with max count of log entries
+ */
+int
+rte_malloc_log_init(uint32_t count)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (mcfg->malloc_log_max) {
+		RTE_LOG(ERR, EAL, "malloc tracking started\n");
+		return -EINVAL;
+	}
+	if (!count || count > MALLOC_TRACKING_MAX) {
+		RTE_LOG(ERR, EAL, "invalid malloc tracking log number\n");
+		return -EINVAL;
+	}
+
+	/* allocate logs. */
+	mcfg->malloc_logs = rte_calloc("malloc_logs", count,
+				       sizeof(*mcfg->malloc_logs), 0);
+	if (!mcfg->malloc_logs)
+		return -ENOMEM;
+
+	mcfg->malloc_log_count = 0;
+	mcfg->malloc_log_max = count;
+	return 0;
+}
+
+/*
+ * Initialize malloc tracking with max count of log entries
+ */
+int
+rte_malloc_log_stop(void)
+{
+	struct malloc_elem *elem;
+	int i;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct rte_malloc_log *log;
+
+	if (!mcfg->malloc_log_max) {
+		RTE_LOG(ERR, EAL, "malloc tracking not started\n");
+		return -EINVAL;
+	}
+
+	/* release all logs. */
+	for (i = 0; i < mcfg->malloc_log_count; i++) {
+		log = &mcfg->malloc_logs[i];
+		/* clear log info from malloc elem. */
+		if (log->type)
+			continue;
+		elem = malloc_elem_from_data(log->addr);
+		if (elem) {
+			elem->log = 0;
+			elem->log_index = 0;
+		}
+	}
+
+	rte_free(mcfg->malloc_logs);
+	mcfg->malloc_logs = NULL;
+	mcfg->malloc_log_max = 0;
+	mcfg->malloc_log_count = 0;
+
+	return 0;
+}
+
+/*
+ * Dump malloc tracking
+ */
+void
+rte_malloc_log_dump(FILE *out, uint32_t detail)
+{
+	struct rte_malloc_log *log;
+	int i;
+	uint32_t n_alloc_free = 0;
+	uint32_t n_alloc = 0;
+	uint32_t n_free_alloc = 0;
+	uint32_t n_free = 0;
+	size_t alloc_leak = 0;
+	size_t free_leak = 0;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	if (!mcfg->malloc_log_max || !mcfg->malloc_logs) {
+		RTE_LOG(ERR, EAL, "malloc tracking not started\n");
+		return;
+	}
+	if (detail)
+		fprintf(out, "Socket  Action Paired            Address    Size Align Mgmt Pad   Total Type\n");
+	for (i = 0; i < mcfg->malloc_log_count; i++) {
+		log = &mcfg->malloc_logs[i];
+		if (detail > 1 ||
+		    (detail == 1 && !log->paired &&
+		     log->type != rte_malloc_log_free))
+			fprintf(out, "%6d %7s %6s %18p %7lu %5lu %4d %3u %7lu %s\n",
+				log->socket,
+				rte_malloc_log_type_name[log->type],
+				log->paired ? "y" : "n",
+				log->addr, log->req_size, log->align,
+				MALLOC_ELEM_OVERHEAD,
+				log->pad,
+				log->size,
+				log->name);
+		if (log->type == rte_malloc_log_free) {
+			n_free++;
+			if (log->paired)
+				n_free_alloc++;
+			else
+				free_leak += log->size;
+		} else {
+			n_alloc++;
+			if (log->paired)
+				n_alloc_free++;
+			else
+				alloc_leak += log->size;
+		}
+	}
+	fprintf(out, "Total rte_malloc and rte_realloc: %u/%u leak %zu(B), rte_free: %u/%u leak: %zu(B)\n",
+		n_alloc_free, n_alloc, alloc_leak,
+		n_free_alloc, n_free, free_leak);
+	if (mcfg->malloc_log_count == mcfg->malloc_log_max)
+		fprintf(out, "Warning: log full, please consider larger log buffer\n");
+}
+
 /* Free the memory space back to heap */
 void rte_free(void *addr)
 {
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+	uint32_t log_malloc, log_index;
+	size_t size;
+
 	if (addr == NULL) return;
-	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
+	if (elem) {
+		log_malloc = elem->log;
+		log_index = elem->log_index;
+		size = elem->size + elem->pad;
+		if (malloc_heap_free(elem) < 0)
+			RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
+		else
+			rte_free_log(addr, log_malloc, log_index, size);
+	} else {
 		RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
+	}
 }
 
 /*
@@ -44,6 +278,8 @@ void rte_free(void *addr)
 rte_malloc_socket(const char *type, size_t size, unsigned int align,
 		int socket_arg)
 {
+	void *ret;
+
 	/* return NULL if size is 0 or alignment is not power-of-2 */
 	if (size == 0 || (align && !rte_is_power_of_2(align)))
 		return NULL;
@@ -57,8 +293,13 @@ void rte_free(void *addr)
 				!rte_eal_has_hugepages())
 		socket_arg = SOCKET_ID_ANY;
 
-	return malloc_heap_alloc(type, size, socket_arg, 0,
+	ret = malloc_heap_alloc(type, size, socket_arg, 0,
 			align == 0 ? 1 : align, 0, false);
+
+	if (ret)
+		rte_malloc_log(type, size, align == 0 ? 1 : align, ret);
+
+	return ret;
 }
 
 /*
@@ -123,10 +364,12 @@ void rte_free(void *addr)
 void *
 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket)
 {
+	struct malloc_elem *elem = malloc_elem_from_data(ptr);
+	uint32_t log_malloc, log_index;
+
 	if (ptr == NULL)
 		return rte_malloc_socket(NULL, size, align, socket);
 
-	struct malloc_elem *elem = malloc_elem_from_data(ptr);
 	if (elem == NULL) {
 		RTE_LOG(ERR, EAL, "Error: memory corruption detected\n");
 		return NULL;
@@ -139,9 +382,15 @@ void rte_free(void *addr)
 	 */
 	if ((socket == SOCKET_ID_ANY ||
 	     (unsigned int)socket == elem->heap->socket_id) &&
-			RTE_PTR_ALIGN(ptr, align) == ptr &&
-			malloc_heap_resize(elem, size) == 0)
-		return ptr;
+	    RTE_PTR_ALIGN(ptr, align) == ptr) {
+		log_malloc = elem->log;
+		log_index = elem->log_index;
+		if (malloc_heap_resize(elem, size) == 0) {
+			rte_realloc_log(log_malloc, log_index, ptr, size,
+					align);
+			return ptr;
+		}
+	}
 
 	/* either requested socket id doesn't match, alignment is off
 	 * or we have no room to expand,
diff --git a/lib/librte_eal/include/rte_malloc.h b/lib/librte_eal/include/rte_malloc.h
index 42ca051..4947ba9 100644
--- a/lib/librte_eal/include/rte_malloc.h
+++ b/lib/librte_eal/include/rte_malloc.h
@@ -508,7 +508,44 @@ struct rte_malloc_socket_stats {
  *   to dump all objects.
  */
 void
-rte_malloc_dump_stats(FILE *f, const char *type);
+rte_malloc_dump_stats(FILE *out, const char *type);
+
+/**
+ * Initialize malloc tracking log buffer.
+ *
+ * @param count
+ *   Max count of tracking log entries, range (1 : 1 << 29)
+ * @return
+ *   0 on success, negative errno otherwise
+ */
+__rte_experimental
+int
+rte_malloc_log_init(uint32_t count);
+
+/**
+ * Stop malloc tracking log.
+ *
+ * @return
+ *   0 on success, negative errno otherwise
+ */
+__rte_experimental
+int
+rte_malloc_log_stop(void);
+
+/**
+ * Dump malloc tracking log to output.
+ *
+ * @param out
+ *   output file handle
+ * @param detail
+ *   detail level of output
+ *   0: summary
+ *   1: potential leaks - allocated without free
+ *   2: all entries
+ */
+__rte_experimental
+void
+rte_malloc_log_dump(FILE *out, uint32_t detail);
 
 /**
  * Dump contents of all malloc heaps to a file.
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index f9ede5b..5151057 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -338,4 +338,7 @@ EXPERIMENTAL {
 
 	# added in 20.05
 	rte_log_can_log;
+	rte_malloc_log_init;
+	rte_malloc_log_dump;
+	rte_malloc_log_stop;
 };
-- 
1.8.3.1

[dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free tracking log

[Xueming Li]

New CLI commands to manipulate malloc tracking log:

dump_malloc start <n>: start malloc tracking with number of buffers
dump_malloc dump <n>:  dump mmalloc tracking log with detail level
dump_malloc stop:      stop malloc tracking

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 app/test-pmd/cmdline.c                      | 60 ++++++++++++++++++++++++++++-
 doc/guides/testpmd_app_ug/testpmd_funcs.rst | 15 ++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index 863b567..0490823 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -9554,6 +9554,8 @@ struct cmd_rm_mirror_rule_result {
 
 struct cmd_dump_result {
 	cmdline_fixed_string_t dump;
+	cmdline_fixed_string_t cmd;
+	uint32_t val;
 };
 
 static void
@@ -9628,6 +9630,16 @@ static void cmd_dump_parsed(void *parsed_result,
 		dump_socket_mem(stdout);
 	else if (!strcmp(res->dump, "dump_memzone"))
 		rte_memzone_dump(stdout);
+	else if (!strcmp(res->dump, "dump_malloc")) {
+		if (!strcmp(res->cmd, "start")) {
+			if (rte_malloc_log_init(res->val))
+				fprintf(stdout, "Failed to start logging\n");
+		} else if (!strcmp(res->cmd, "dump")) {
+			rte_malloc_log_dump(stdout, res->val);
+		} else if (!strcmp(res->cmd, "stop")) {
+			rte_malloc_log_stop();
+		}
+	}
 	else if (!strcmp(res->dump, "dump_struct_sizes"))
 		dump_struct_sizes();
 	else if (!strcmp(res->dump, "dump_ring"))
@@ -9650,7 +9662,50 @@ static void cmd_dump_parsed(void *parsed_result,
 		"dump_mempool#"
 		"dump_devargs#"
 		"dump_log_types");
-
+cmdline_parse_token_string_t cmd_dump_malloc =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump,
+		"dump_malloc");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_start =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"start");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_dump =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"dump");
+cmdline_parse_token_string_t cmd_dump_malloc_cmd_stop =
+	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, cmd,
+		"stop");
+cmdline_parse_token_num_t cmd_dump_val =
+	TOKEN_NUM_INITIALIZER(struct cmd_dump_result, val, UINT32);
+
+cmdline_parse_inst_t cmd_dump_malloc_start = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Start rte_malloc tracking log with <count> buffer entries",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_start,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_dump = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Dump rte_malloc tracking log with <level>, 0:summary, 1:leaks, 2: all",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_dump,
+		(void *)&cmd_dump_val,
+		NULL,
+	},
+};
+cmdline_parse_inst_t cmd_dump_malloc_stop = {
+	.f = cmd_dump_parsed,  /* function to call */
+	.help_str = "Stop rte_malloc tracking log",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dump_malloc,
+		(void *)&cmd_dump_malloc_cmd_stop,
+		NULL,
+	},
+};
 cmdline_parse_inst_t cmd_dump = {
 	.f = cmd_dump_parsed,  /* function to call */
 	.data = NULL,      /* 2nd arg of func */
@@ -19505,6 +19560,9 @@ struct cmd_showport_macs_result {
 	(cmdline_parse_inst_t *)&cmd_showport_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_config_rss_hash_key,
 	(cmdline_parse_inst_t *)&cmd_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_start,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_dump,
+	(cmdline_parse_inst_t *)&cmd_dump_malloc_stop,
 	(cmdline_parse_inst_t *)&cmd_dump_one,
 	(cmdline_parse_inst_t *)&cmd_ethertype_filter,
 	(cmdline_parse_inst_t *)&cmd_syn_filter,
diff --git a/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
index dcee5de..1c55f23 100644
--- a/doc/guides/testpmd_app_ug/testpmd_funcs.rst
+++ b/doc/guides/testpmd_app_ug/testpmd_funcs.rst
@@ -567,6 +567,21 @@ Dumps the statistics of all or specific memory pool::
 
    testpmd> dump_mempool [mempool_name]
 
+dump malloc
+~~~~~~~~~~~~
+
+Start rte_malloc and rte_free tracking with number of buffers::
+
+   testpmd> dump_malloc start <count>
+
+Dump tracking result with different level, 0:summary, 1:leaks, 2: all::
+
+   testpmd> dump_malloc dump <level>
+
+Stop tracking::
+
+   testpmd> dump_malloc stop
+
 dump devargs
 ~~~~~~~~~~~~
 
-- 
1.8.3.1

Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free tracking log

[Iremonger, Bernard]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Friday, April 17, 2020 9:04 AM
> To: Burakov, Anatoly <anatoly.burakov@intel.com>; Yigit, Ferruh
> <ferruh.yigit@intel.com>; Stephen Hemminger
> <stephen@networkplumber.org>
> Cc: dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> Subject: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free
> tracking log
> 
> New CLI commands to manipulate malloc tracking log:
> 
> dump_malloc start <n>: start malloc tracking with number of buffers
> dump_malloc dump <n>:  dump mmalloc tracking log with detail level
> dump_malloc stop:      stop malloc tracking
> 
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>

Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>

Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free tracking log

[Somnath Kotur]

On Tue, Apr 21, 2020 at 7:11 PM Iremonger, Bernard
<bernard.iremonger@intel.com> wrote:
>
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > Sent: Friday, April 17, 2020 9:04 AM
> > To: Burakov, Anatoly <anatoly.burakov@intel.com>; Yigit, Ferruh
> > <ferruh.yigit@intel.com>; Stephen Hemminger
> > <stephen@networkplumber.org>
> > Cc: dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> > Subject: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free
> > tracking log
> >
> > New CLI commands to manipulate malloc tracking log:
> >
> > dump_malloc start <n>: start malloc tracking with number of buffers
> > dump_malloc dump <n>:  dump mmalloc tracking log with detail level
> > dump_malloc stop:      stop malloc tracking
> >
> > Signed-off-by: Xueming Li <xuemingl@mellanox.com>
>
> Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>

Sorry if this has been answered already, but just wanted to know if
there is any reason this is not yet merged in ...could be a useful
tool to debug leaks in PMDs
Is there any other /better way to identify memory leaks now which is
why this is no longer under consideration ?

Thanks
som

Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free tracking log

[Xueming(Steven) Li]

Hi Som,

> -----Original Message-----
> From: Somnath Kotur <somnath.kotur@broadcom.com>
> Sent: Thursday, July 30, 2020 11:10 PM
> To: Iremonger, Bernard <bernard.iremonger@intel.com>
> Cc: Xueming(Steven) Li <xuemingl@mellanox.com>; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>; Stephen
> Hemminger <stephen@networkplumber.org>; dev@dpdk.org; Asaf Penso
> <asafp@mellanox.com>
> Subject: Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free
> tracking log
> 
> On Tue, Apr 21, 2020 at 7:11 PM Iremonger, Bernard
> <bernard.iremonger@intel.com> wrote:
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > Sent: Friday, April 17, 2020 9:04 AM
> > > To: Burakov, Anatoly <anatoly.burakov@intel.com>; Yigit, Ferruh
> > > <ferruh.yigit@intel.com>; Stephen Hemminger
> > > <stephen@networkplumber.org>
> > > Cc: dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> > > Subject: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and
> > > free tracking log
> > >
> > > New CLI commands to manipulate malloc tracking log:
> > >
> > > dump_malloc start <n>: start malloc tracking with number of buffers
> > > dump_malloc dump <n>:  dump mmalloc tracking log with detail level
> > > dump_malloc stop:      stop malloc tracking
> > >
> > > Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> >
> > Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>
> 
> Sorry if this has been answered already, but just wanted to know if there is any
> reason this is not yet merged in ...could be a useful tool to debug leaks in PMDs
> Is there any other /better way to identify memory leaks now which is why this
> is no longer under consideration ?

AFAIK, new tracking point system is introduced to record key event not only memory.
So this patchset no longer valid.
 
> 
> Thanks
> som

Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free tracking log

[Somnath Kotur]

On Thu, Jul 30, 2020 at 8:43 PM Xueming(Steven) Li
<xuemingl@mellanox.com> wrote:
>
> Hi Som,
>
> > -----Original Message-----
> > From: Somnath Kotur <somnath.kotur@broadcom.com>
> > Sent: Thursday, July 30, 2020 11:10 PM
> > To: Iremonger, Bernard <bernard.iremonger@intel.com>
> > Cc: Xueming(Steven) Li <xuemingl@mellanox.com>; Burakov, Anatoly
> > <anatoly.burakov@intel.com>; Yigit, Ferruh <ferruh.yigit@intel.com>; Stephen
> > Hemminger <stephen@networkplumber.org>; dev@dpdk.org; Asaf Penso
> > <asafp@mellanox.com>
> > Subject: Re: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and free
> > tracking log
> >
> > On Tue, Apr 21, 2020 at 7:11 PM Iremonger, Bernard
> > <bernard.iremonger@intel.com> wrote:
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > Sent: Friday, April 17, 2020 9:04 AM
> > > > To: Burakov, Anatoly <anatoly.burakov@intel.com>; Yigit, Ferruh
> > > > <ferruh.yigit@intel.com>; Stephen Hemminger
> > > > <stephen@networkplumber.org>
> > > > Cc: dev@dpdk.org; Asaf Penso <asafp@mellanox.com>
> > > > Subject: [dpdk-dev] [RFC v2 2/2] app/testpmd: support malloc and
> > > > free tracking log
> > > >
> > > > New CLI commands to manipulate malloc tracking log:
> > > >
> > > > dump_malloc start <n>: start malloc tracking with number of buffers
> > > > dump_malloc dump <n>:  dump mmalloc tracking log with detail level
> > > > dump_malloc stop:      stop malloc tracking
> > > >
> > > > Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> > >
> > > Acked-by: Bernard Iremonger <bernard.iremonger@intel.com>
> >
> > Sorry if this has been answered already, but just wanted to know if there is any
> > reason this is not yet merged in ...could be a useful tool to debug leaks in PMDs
> > Is there any other /better way to identify memory leaks now which is why this
> > is no longer under consideration ?
>
> AFAIK, new tracking point system is introduced to record key event not only memory.
> So this patchset no longer valid.
Hmm sorry didn't get that ..did you mean tracing system ? Could you
please elaborate and provide any pointers on setting this up for
monitoring memory?
Any pointers would be greatly appreciated
>
> >
> > Thanks
> > som
>
>