[dpdk-dev] [PATCH v2 14/41] net/mlx5: add open IB device routines

[Srikanth Kaka <srikanth.k@oneconvergence.com>]

The mlx5_os_open_device(), mlx5_config_doorbell_mapping_env(),
mlx5_restore_doorbell_mapping_env(), mlx5_alloc_verbs_buf() and
mlx5_free_verbs_buf() are equivalent to Linux APIs of the same
name

Signed-off-by: Srikanth Kaka <srikanth.k@oneconvergence.com>
Signed-off-by: Vag Singh <vag.singh@oneconvergence.com>
Signed-off-by: Anand Thulasiram <avelu@juniper.net>
---
 drivers/net/mlx5/freebsd/mlx5_os.c | 155 +++++++++++++++++++++++++++++
 1 file changed, 155 insertions(+)

diff --git a/drivers/net/mlx5/freebsd/mlx5_os.c b/drivers/net/mlx5/freebsd/mlx5_os.c
index 1d8e627988..53f74b9d85 100644
--- a/drivers/net/mlx5/freebsd/mlx5_os.c
+++ b/drivers/net/mlx5/freebsd/mlx5_os.c
@@ -5,16 +5,28 @@
  */
 
 #include <stddef.h>
+#include <unistd.h>
+#include <string.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <errno.h>
 
+#include <rte_malloc.h>
+#include <ethdev_driver.h>
+#include <rte_eal_paging.h>
+
 #include <mlx5_glue.h>
+#include <mlx5_devx_cmds.h>
 #include <mlx5_common.h>
+#include <mlx5_common_mr.h>
 
 #include "mlx5_defs.h"
 #include "mlx5.h"
+#include "mlx5_common_os.h"
+#include "mlx5_rxtx.h"
 #include "mlx5_autoconf.h"
+#include "mlx5_mr.h"
+#include "mlx5_verbs.h"
 
 /**
  * Get mlx5 device attributes. The glue function query_device_ex() is called
@@ -86,6 +98,149 @@ mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr)
 	return err;
 }
 
+/**
+ * Verbs callback to allocate a memory. This function should allocate the space
+ * according to the size provided residing inside a huge page.
+ * Please note that all allocation must respect the alignment from libmlx5
+ * (i.e. currently rte_mem_page_size()).
+ *
+ * @param[in] size
+ *   The size in bytes of the memory to allocate.
+ * @param[in] data
+ *   A pointer to the callback data.
+ *
+ * @return
+ *   Allocated buffer, NULL otherwise and rte_errno is set.
+ */
+static void *
+mlx5_alloc_verbs_buf(size_t size, void *data)
+{
+	struct mlx5_dev_ctx_shared *sh = data;
+	void *ret;
+	size_t alignment = rte_mem_page_size();
+	if (alignment == (size_t)-1) {
+		DRV_LOG(ERR, "Failed to get mem page size");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	MLX5_ASSERT(data != NULL);
+	ret = mlx5_malloc(0, size, alignment, sh->numa_node);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+/**
+ * Verbs callback to free a memory.
+ *
+ * @param[in] ptr
+ *   A pointer to the memory to free.
+ * @param[in] data
+ *   A pointer to the callback data.
+ */
+static void
+mlx5_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	MLX5_ASSERT(data != NULL);
+	mlx5_free(ptr);
+}
+
+static int
+mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
+{
+	char *env;
+	int value;
+
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	/* Get environment variable to store. */
+	env = getenv(MLX5_SHUT_UP_BF);
+	value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET;
+	if (config->dbnc == MLX5_ARG_UNSET)
+		setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1);
+	else
+		setenv(MLX5_SHUT_UP_BF,
+		       config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1);
+	return value;
+}
+
+static void
+mlx5_restore_doorbell_mapping_env(int value)
+{
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	/* Restore the original environment variable state. */
+	if (value == MLX5_ARG_UNSET)
+		unsetenv(MLX5_SHUT_UP_BF);
+	else
+		setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1);
+}
+
+/**
+ * Function API to open IB device.
+ *
+ * This function calls the Linux glue APIs to open a device.
+ *
+ * @param[in] spawn
+ *   Pointer to the IB device attributes (name, port, etc).
+ * @param[out] config
+ *   Pointer to device configuration structure.
+ * @param[out] sh
+ *   Pointer to shared context structure.
+ *
+ * @return
+ *   0 on success, a positive error value otherwise.
+ */
+int
+mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
+		     const struct mlx5_dev_config *config,
+		     struct mlx5_dev_ctx_shared *sh)
+{
+	int dbmap_env;
+	int err = 0;
+
+	pthread_mutex_init(&sh->txpp.mutex, NULL);
+	/*
+	 * Configure environment variable "MLX5_BF_SHUT_UP"
+	 * before the device creation. The rdma_core library
+	 * checks the variable at device creation and
+	 * stores the result internally.
+	 */
+	dbmap_env = mlx5_config_doorbell_mapping_env(config);
+	/* Try to open IB device with DV first, then usual Verbs. */
+	errno = 0;
+	sh->ctx = mlx5_glue->dv_open_device(spawn->phys_dev);
+	if (sh->ctx) {
+		sh->devx = 1;
+		DRV_LOG(DEBUG, "DevX is supported");
+		/* The device is created, no need for environment. */
+		mlx5_restore_doorbell_mapping_env(dbmap_env);
+	} else {
+		/* The environment variable is still configured. */
+		sh->ctx = mlx5_glue->open_device(spawn->phys_dev);
+		err = errno ? errno : ENODEV;
+		/*
+		 * The environment variable is not needed anymore,
+		 * all device creation attempts are completed.
+		 */
+		mlx5_restore_doorbell_mapping_env(dbmap_env);
+		if (!sh->ctx)
+			return err;
+		DRV_LOG(DEBUG, "DevX is NOT supported");
+		err = 0;
+	}
+	if (!err && sh->ctx) {
+		/* Hint libmlx5 to use PMD allocator for data plane resources */
+		mlx5_glue->dv_set_context_attr(sh->ctx,
+			MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
+			(void *)((uintptr_t)&(struct mlx5dv_ctx_allocators){
+				.alloc = &mlx5_alloc_verbs_buf,
+				.free = &mlx5_free_verbs_buf,
+				.data = sh,
+			}));
+	}
+	return err;
+}
+
 /**
  * Set the reg_mr and dereg_mr call backs
  *
-- 
2.30.2