[dpdk-dev] [PATCH v5 05/10] app/test: support resources archived by tar

[Jan Viktorin <viktorin@rehivetech.com>]

When a more complex resource (a file hierarchy) is needed, packing
every single file as a single resource would be very ineffective. For
that purpose, it is possible to pack the files into a tar archive,
extract it before test from the resource and finally clean up all the
created files.

This patch introduces functions resource_untar and resource_rm_by_tar
to perform those tasks. An example of using those functions is included
as a test.

A new dependency is required to build the app/test: libarchive.

Signed-off-by: Jan Viktorin <viktorin@rehivetech.com>
---
v5:
* fix archive_read_data_block, more info:
  https://github.com/libarchive/libarchive/blob/master/examples/untar.c#L205
  http://markmail.org/message/y4pgi44gqjd4jfuc
---
 app/test/Makefile        |   9 +++
 app/test/resource.c      | 200 +++++++++++++++++++++++++++++++++++++++++++++++
 app/test/resource.h      |  13 +++
 app/test/test_resource.c |  29 +++++++
 4 files changed, 251 insertions(+)

diff --git a/app/test/Makefile b/app/test/Makefile
index fe67eee..1cc6258 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -51,6 +51,13 @@ $(1).res.o: $(2)
 		/dev/stdin $$@ < $$<
 endef
 
+define linked_tar_resource
+$(1).tar: $(2)
+	tar -C $$(dir $$<) -cf $$@ $$(notdir $$<)
+
+$(call linked_resource,$(1),$(1).tar)
+endef
+
 #
 # library name
 #
@@ -64,6 +71,7 @@ SRCS-y += test.c
 SRCS-y += resource.c
 SRCS-y += test_resource.c
 $(eval $(call linked_resource,test_resource_c,resource.c))
+$(eval $(call linked_tar_resource,test_resource_tar,test_resource.c))
 SRCS-y += test_pci.c
 SRCS-y += test_prefetch.c
 SRCS-y += test_byteorder.c
@@ -185,6 +193,7 @@ CFLAGS += $(WERROR_FLAGS)
 CFLAGS += -D_GNU_SOURCE
 
 LDLIBS += -lm
+LDLIBS += -larchive
 
 # Disable VTA for memcpy test
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
diff --git a/app/test/resource.c b/app/test/resource.c
index acb63c1..8c42eea 100644
--- a/app/test/resource.c
+++ b/app/test/resource.c
@@ -33,6 +33,8 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <archive.h>
+#include <archive_entry.h>
 #include <errno.h>
 #include <sys/queue.h>
 
@@ -95,6 +97,204 @@ int resource_fwrite_file(const struct resource *r, const char *fname)
 	return ret;
 }
 
+static int do_copy(struct archive *r, struct archive *w)
+{
+	const void *buf;
+	size_t len;
+#if ARCHIVE_VERSION_NUMBER >= 3000000
+	int64_t off;
+#else
+	off_t off;
+#endif
+	int ret;
+
+	while (1) {
+		ret = archive_read_data_block(r, &buf, &len, &off);
+		if (ret == ARCHIVE_RETRY)
+			continue;
+
+		if (ret == ARCHIVE_EOF)
+			return 0;
+
+		if (ret != ARCHIVE_OK)
+			return ret;
+
+		do {
+			ret = archive_write_data_block(w, buf, len, off);
+			if (ret != ARCHIVE_OK && ret != ARCHIVE_RETRY)
+				return ret;
+		} while (ret != ARCHIVE_OK);
+	}
+}
+
+int resource_untar(const struct resource *res)
+{
+	struct archive *r;
+	struct archive *w;
+	struct archive_entry *e;
+	void *p;
+	int flags = 0;
+	int ret;
+
+	p = malloc(resource_size(res));
+	if (p == NULL)
+		rte_panic("Failed to malloc %zu B\n", resource_size(res));
+
+	memcpy(p, res->begin, resource_size(res));
+
+	r = archive_read_new();
+	if (r == NULL) {
+		free(p);
+		return -1;
+	}
+
+	archive_read_support_format_all(r);
+	archive_read_support_filter_all(r);
+
+	w = archive_write_disk_new();
+	if (w == NULL) {
+		archive_read_free(r);
+		free(p);
+		return -1;
+	}
+
+	flags |= ARCHIVE_EXTRACT_PERM;
+	flags |= ARCHIVE_EXTRACT_FFLAGS;
+	archive_write_disk_set_options(w, flags);
+	archive_write_disk_set_standard_lookup(w);
+
+	ret = archive_read_open_memory(r, p, resource_size(res));
+	if (ret != ARCHIVE_OK)
+		goto fail;
+
+	while (1) {
+		ret = archive_read_next_header(r, &e);
+		if (ret == ARCHIVE_EOF)
+			break;
+		if (ret != ARCHIVE_OK)
+			goto fail;
+
+		ret = archive_write_header(w, e);
+		if (ret == ARCHIVE_EOF)
+			break;
+		if (ret != ARCHIVE_OK)
+			goto fail;
+
+		if (archive_entry_size(e) == 0)
+			continue;
+
+		ret = do_copy(r, w);
+		if (ret != ARCHIVE_OK)
+			goto fail;
+
+		ret = archive_write_finish_entry(w);
+		if (ret != ARCHIVE_OK)
+			goto fail;
+	}
+
+	archive_write_free(w);
+	archive_read_free(r);
+	free(p);
+	return 0;
+
+fail:
+	archive_write_free(w);
+	archive_read_free(r);
+	free(p);
+	rte_panic("Failed: %s\n", archive_error_string(r));
+	return -1;
+}
+
+int resource_rm_by_tar(const struct resource *res)
+{
+	struct archive *r;
+	struct archive_entry *e;
+	void *p;
+	int try_again = 1;
+	int attempts = 0;
+	int ret;
+
+	p = malloc(resource_size(res));
+	if (p == NULL)
+		rte_panic("Failed to malloc %zu B\n", resource_size(res));
+
+	memcpy(p, res->begin, resource_size(res));
+
+	/*
+	 * If somebody creates a file somewhere inside the extracted TAR
+	 * hierarchy during a test the resource_rm_by_tar might loop
+	 * infinitely. We prevent this by adding the attempts counter there.
+	 * In normal case, max N iteration is done where N is the depth of
+	 * the file-hierarchy.
+	 */
+	while (try_again && attempts < 10000) {
+		r = archive_read_new();
+		if (r == NULL) {
+			free(p);
+			return -1;
+		}
+
+		archive_read_support_format_all(r);
+		archive_read_support_filter_all(r);
+
+		ret = archive_read_open_memory(r, p, resource_size(res));
+		if (ret != ARCHIVE_OK) {
+			fprintf(stderr, "Failed: %s\n",
+					archive_error_string(r));
+			goto fail;
+		}
+
+		try_again = 0;
+
+		while (1) {
+			ret = archive_read_next_header(r, &e);
+			if (ret == ARCHIVE_EOF)
+				break;
+			if (ret != ARCHIVE_OK)
+				goto fail;
+
+			ret = remove(archive_entry_pathname(e));
+			if (ret < 0) {
+				switch (errno) {
+				case ENOTEMPTY:
+				case EEXIST:
+					try_again = 1;
+					break;
+
+				/* should not usually happen: */
+				case ENOENT:
+				case ENOTDIR:
+				case EROFS:
+					attempts += 1;
+					continue;
+				default:
+					perror("Failed to remove file");
+					goto fail;
+				}
+			}
+		}
+
+		archive_read_free(r);
+		attempts += 1;
+	}
+
+	if (attempts >= 10000) {
+		fprintf(stderr, "Failed to remove archive\n");
+		free(p);
+		return -1;
+	}
+
+	free(p);
+	return 0;
+
+fail:
+	archive_read_free(r);
+	free(p);
+
+	rte_panic("Failed: %s\n", archive_error_string(r));
+	return -1;
+}
+
 void resource_register(struct resource *r)
 {
 	TAILQ_INSERT_TAIL(&resource_list, r, next);
diff --git a/app/test/resource.h b/app/test/resource.h
index ac9cae6..1e96122 100644
--- a/app/test/resource.h
+++ b/app/test/resource.h
@@ -89,6 +89,19 @@ int resource_fwrite(const struct resource *r, FILE *f);
 int resource_fwrite_file(const struct resource *r, const char *fname);
 
 /**
+ * Treat the given resource as a tar archive. Extract
+ * the archive to the current directory.
+ */
+int resource_untar(const struct resource *res);
+
+/**
+ * Treat the given resource as a tar archive. Remove
+ * all files (related to the current directory) listed
+ * in the tar archive.
+ */
+int resource_rm_by_tar(const struct resource *res);
+
+/**
  * Register a resource in the global list of resources.
  * Not intended for direct use, please check the REGISTER_RESOURCE
  * macro.
diff --git a/app/test/test_resource.c b/app/test/test_resource.c
index 3d1bf00..1e85040 100644
--- a/app/test/test_resource.c
+++ b/app/test/test_resource.c
@@ -85,6 +85,32 @@ static int test_resource_c(void)
 	return 0;
 }
 
+REGISTER_LINKED_RESOURCE(test_resource_tar);
+
+static int test_resource_tar(void)
+{
+	const struct resource *r;
+	FILE *f;
+
+	r = resource_find("test_resource_tar");
+	TEST_ASSERT_NOT_NULL(r, "No test_resource_tar found");
+	TEST_ASSERT(!strcmp(r->name, "test_resource_tar"),
+			"Found resource %s, expected test_resource_tar",
+			r->name);
+
+	TEST_ASSERT_SUCCESS(resource_untar(r),
+			"Failed to to untar %s", r->name);
+
+	f = fopen("test_resource.c", "r");
+	TEST_ASSERT_NOT_NULL(f,
+			"Missing extracted file test_resource.c");
+	fclose(f);
+
+	TEST_ASSERT_SUCCESS(resource_rm_by_tar(r),
+			"Failed to remove extracted contents of %s", r->name);
+	return 0;
+}
+
 static int test_resource(void)
 {
 	if (test_resource_dpdk())
@@ -93,6 +119,9 @@ static int test_resource(void)
 	if (test_resource_c())
 		return -1;
 
+	if (test_resource_tar())
+		return -1;
+
 	return 0;
 }
 
-- 
2.8.0