Re: [dpdk-dev] [PATCH v2 2/2] examples/vdpa: introduce a new sample for vDPA

["Wang, Xiao W" <xiao.w.wang@intel.com>]

Hi Xiaolong,

> -----Original Message-----
> From: Ye, Xiaolong
> Sent: Friday, September 14, 2018 2:07 AM
> To: dev@dpdk.org; Maxime Coquelin <maxime.coquelin@redhat.com>; Bie,
> Tiwei <tiwei.bie@intel.com>; Wang, Zhihong <zhihong.wang@intel.com>
> Cc: Wang, Xiao W <xiao.w.wang@intel.com>; Rami Rosen
> <roszenrami@gmail.com>; Wang, Haiyue <haiyue.wang@intel.com>; Ye,
> Xiaolong <xiaolong.ye@intel.com>
> Subject: [PATCH v2 2/2] examples/vdpa: introduce a new sample for vDPA
> 
> The vdpa sample application creates vhost-user sockets by using the
> vDPA backend. vDPA stands for vhost Data Path Acceleration which utilizes
> virtio ring compatible devices to serve virtio driver directly to enable
> datapath acceleration. As vDPA driver can help to set up vhost datapath,
> this application doesn't need to launch dedicated worker threads for vhost
> enqueue/dequeue operations.
> 
> Signed-off-by: Xiao Wang <xiao.w.wang@intel.com>
> Signed-off-by: Xiaolong Ye <xiaolong.ye@intel.com>
> ---
> 
> v2 changes:
> 
> * fix a compilation error reported by Rosen
> * improve create cmd in interactive mode and add two new cmds: list,
> * quit
> * add application documentation
> 
>  MAINTAINERS                        |   2 +
>  doc/guides/sample_app_ug/index.rst |   1 +
>  doc/guides/sample_app_ug/vdpa.rst  | 115 ++++++++
>  examples/Makefile                  |   2 +-
>  examples/vdpa/Makefile             |  32 +++
>  examples/vdpa/main.c               | 437 +++++++++++++++++++++++++++++
>  examples/vdpa/meson.build          |  16 ++
>  7 files changed, 604 insertions(+), 1 deletion(-)
>  create mode 100644 doc/guides/sample_app_ug/vdpa.rst
>  create mode 100644 examples/vdpa/Makefile
>  create mode 100644 examples/vdpa/main.c
>  create mode 100644 examples/vdpa/meson.build
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 9fd258fad..f84dbf2a7 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -682,6 +682,8 @@ F: doc/guides/sample_app_ug/vhost.rst
>  F: examples/vhost_scsi/
>  F: doc/guides/sample_app_ug/vhost_scsi.rst
>  F: examples/vhost_crypto/
> +F: examples/vdpa/
> +F: doc/guides/sample_app_ug/vdpa.rst
> 
>  Vhost PMD
>  M: Maxime Coquelin <maxime.coquelin@redhat.com>
> diff --git a/doc/guides/sample_app_ug/index.rst
> b/doc/guides/sample_app_ug/index.rst
> index 5bedf4f6f..74b12af85 100644
> --- a/doc/guides/sample_app_ug/index.rst
> +++ b/doc/guides/sample_app_ug/index.rst
> @@ -45,6 +45,7 @@ Sample Applications User Guides
>      vhost
>      vhost_scsi
>      vhost_crypto
> +    vdpa
>      netmap_compatibility
>      ip_pipeline
>      test_pipeline
> diff --git a/doc/guides/sample_app_ug/vdpa.rst
> b/doc/guides/sample_app_ug/vdpa.rst
> new file mode 100644
> index 000000000..ab222731e
> --- /dev/null
> +++ b/doc/guides/sample_app_ug/vdpa.rst
> @@ -0,0 +1,115 @@
> +..  SPDX-License-Identifier: BSD-3-Clause
> +    Copyright(c) 2018 Intel Corporation.
> +
> +Vdpa Sample Application
> +=======================
> +
> +The vdpa sample application creates vhost-user sockets by using the
> +vDPA backend. vDPA stands for vhost Data Path Acceleration which utilizes
> +virtio ring compatible devices to serve virtio driver directly to enable
> +datapath acceleration. As vDPA driver can help to set up vhost datapath,
> +this application doesn't need to launch dedicated worker threads for vhost
> +enqueue/dequeue operations.
> +

[...]

> +
> +Take IFCVF driver for example:
> +
> +.. code-block:: console
> +
> +        ./vdpa --log-level=9 -c 0x6 -n 4 --socket-mem 1024,1024 \
> +                -w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1 \
> +                -- --interactive
> +
> +.. note::
> +    We need to bind VFIO-pci to VFs before running vdpa sample.

Replace "VFIO-pci" with "vfio-pci".

> +
> +    * modprobe vfio-pci
> +    * ./usertools/dpdk-devbind.py -b vfio-pci 06:00.2 06:00.3
> +
> +Then we can create 2 vdpa ports in interactive cmdline.
> +
> +.. code-block:: console
> +
> +        vdpa> list
> +        device id       device address
> +        0               0000:06:00.2
> +        1               0000:06:00.3

Could we show out also the device's features and supported queue number?

> +        vdpa> create /tmp/vdpa-socket0 0000:06:00.2
> +        vdpa> create /tmp/vdpa-socket1 0000:06:00.3
> +
> +.. _vdpa_app_run_vm:
> +

[...]

> +#include <stdint.h>
> +#include <string.h>
> +#include <unistd.h>
> +
> +#include <rte_ethdev.h>
> +#include <rte_malloc.h>
> +#include <rte_vhost.h>
> +#include <rte_vdpa.h>
> +#include <rte_pci.h>
> +
> +#include <cmdline_parse.h>
> +#include <cmdline_socket.h>
> +#include <cmdline_parse_string.h>
> +#include <cmdline.h>
> +
> +#define MAX_PATH_LEN 128
> +#define MAX_VDPA_SAMPLE_PORTS 1024

[...]

> +	ret = rte_vhost_driver_unregister(socket_path);
> +	if (ret != 0)
> +		RTE_LOG(ERR, USER1,
> +				"Fail to unregister vhost driver for %s.\n",
> +				socket_path);
> +}
> +
> +static void
> +vdpa_sample_quit(void)
> +{
> +	int i;
> +	for (i = 0; i <  RTE_MIN(MAX_VDPA_SAMPLE_PORTS, dev_total); i++) {

Double " ".

> +		if (vports[i].ifname[0] != '\0')
> +			close_vdpa(&vports[i]);
> +	}
> +}
> +
> +static void
> +signal_handler(int signum)
> +{
> +	if (signum == SIGINT || signum == SIGTERM) {
> +		printf("\nSignal %d received, preparing to exit...\n", signum);
> +		vdpa_sample_quit();
> +		exit(0);
> +	}
> +}
> +
> +/* interactive cmds */
> +
> +/* *** Help command with introduction. *** */
> +struct cmd_help_result {
> +	cmdline_fixed_string_t help;
> +};
> +
> +static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
> +		struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	cmdline_printf(
> +		cl,
> +		"\n"
> +		"The following commands are currently available:\n\n"
> +		"Control:\n"
> +		"    help                                      : Show interactive instructions.\n"
> +		"    list                                      : list all available vdpa devices.\n"
> +		"    create <socket file> <vdev addr>          : create a new vdpa
> port.\n"
> +		"    quit                                      : exit vdpa sample app.\n"
> +	);
> +}
> +
> +cmdline_parse_token_string_t cmd_help_help =
> +	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
> +
> +cmdline_parse_inst_t cmd_help = {
> +	.f = cmd_help_parsed,
> +	.data = NULL,
> +	.help_str = "show help",
> +	.tokens = {
> +		(void *)&cmd_help_help,
> +		NULL,
> +	},
> +};
> +
> +/* *** List all available vdpa devices *** */
> +struct cmd_list_result {
> +	cmdline_fixed_string_t action;
> +};
> +
> +static void cmd_list_vdpa_devices_parsed(
> +		__attribute__((unused)) void *parsed_result,
> +		struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	int did;
> +	struct rte_vdpa_device *vdev;
> +	struct rte_pci_addr addr;
> +
> +	cmdline_printf(cl, "device id\tdevice address\n");
> +	for (did = 0; did < dev_total; did++) {
> +		vdev = rte_vdpa_get_device(did);
> +		if (!vdev)
> +			continue;
> +		addr = vdev->addr.pci_addr;
> +		cmdline_printf(cl, "%d\t\t"PCI_PRI_FMT"\n", did, addr.domain,
> +				addr.bus, addr.devid, addr.function);
> +	}
> +}
> +
> +cmdline_parse_token_string_t cmd_action_list =
> +	TOKEN_STRING_INITIALIZER(struct cmd_list_result, action, "list");
> +
> +cmdline_parse_inst_t cmd_list_vdpa_devices = {
> +	.f = cmd_list_vdpa_devices_parsed,  /* function to call */
> +	.data = NULL,      /* 2nd arg of func */
> +	.help_str = "list all available vdpa devices",
> +	.tokens = {        /* token list, NULL terminated */
> +		(void *)&cmd_action_list,
> +		NULL,
> +	},
> +};
> +
> +/* *** Create new vdpa port *** */
> +struct cmd_create_result {
> +	cmdline_fixed_string_t action;
> +	cmdline_fixed_string_t socket_path;
> +	cmdline_fixed_string_t bdf;
> +};
> +
> +static void cmd_create_vdpa_port_parsed(void *parsed_result,
> +		struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	int did;
> +	struct cmd_create_result *res = parsed_result;
> +	struct rte_vdpa_dev_addr addr;
> +
> +	strncpy(vports[devcnt].ifname, res->socket_path, MAX_PATH_LEN);
> +	if (eal_parse_pci_DomBDF(res->bdf, &addr.pci_addr) != 0) {

this is a deprecated API, use rte_pci_addr_parse.

> +		cmdline_printf(cl, "Unable to parse the given bdf.\n");
> +		return;
> +	}
> +	addr.type = PCI_ADDR;
> +	did = rte_vdpa_find_device_id(&addr);
> +	if (did < 0) {
> +		cmdline_printf(cl, "Unable to find vdpa devide id.\n");
> +		return;
> +	}
> +
> +	vports[devcnt].did = did;
> +	start_vdpa(&vports[devcnt]);
> +	devcnt++;
> +}
> +
> +cmdline_parse_token_string_t cmd_action_create =
> +	TOKEN_STRING_INITIALIZER(struct cmd_create_result, action,
> "create");
> +cmdline_parse_token_string_t cmd_socket_path =
> +	TOKEN_STRING_INITIALIZER(struct cmd_create_result, socket_path,
> NULL);
> +cmdline_parse_token_string_t cmd_bdf =
> +	TOKEN_STRING_INITIALIZER(struct cmd_create_result, bdf, NULL);
> +
> +cmdline_parse_inst_t cmd_create_vdpa_port = {
> +	.f = cmd_create_vdpa_port_parsed,  /* function to call */
> +	.data = NULL,      /* 2nd arg of func */
> +	.help_str = "create a new vdpa port",
> +	.tokens = {        /* token list, NULL terminated */
> +		(void *)&cmd_action_create,
> +		(void *)&cmd_socket_path,
> +		(void *)&cmd_bdf,
> +		NULL,
> +	},
> +};
> +
> +/* *** QUIT *** */
> +struct cmd_quit_result {
> +	cmdline_fixed_string_t quit;
> +};
> +
> +static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
> +		struct cmdline *cl,
> +		__attribute__((unused)) void *data)
> +{
> +	vdpa_sample_quit();
> +	cmdline_quit(cl);
> +}
> +
> +cmdline_parse_token_string_t cmd_quit_quit =
> +	TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
> +
> +cmdline_parse_inst_t cmd_quit = {
> +	.f = cmd_quit_parsed,
> +	.data = NULL,
> +	.help_str = "quit: exit application",
> +	.tokens = {
> +		(void *)&cmd_quit_quit,
> +		NULL,
> +	},
> +};
> +cmdline_parse_ctx_t main_ctx[] = {
> +	(cmdline_parse_inst_t *)&cmd_help,
> +	(cmdline_parse_inst_t *)&cmd_list_vdpa_devices,
> +	(cmdline_parse_inst_t *)&cmd_create_vdpa_port,
> +	(cmdline_parse_inst_t *)&cmd_quit,
> +	NULL,
> +};
> +
> +int
> +main(int argc, char *argv[])
> +{
> +	char ch;
> +	int i;
> +	int ret;
> +	struct cmdline *cl;
> +
> +	ret = rte_eal_init(argc, argv);
> +	if (ret < 0)
> +		rte_exit(EXIT_FAILURE, "eal init failed\n");
> +	argc -= ret;
> +	argv += ret;
> +
> +	signal(SIGINT, signal_handler);
> +	signal(SIGTERM, signal_handler);
> +
> +	data_init();

data_init first, then set signal handler, otherwise the signal handler may use some uninitialized variable.

> +
> +	ret = parse_args(argc, argv);
> +	if (ret < 0)
> +		rte_exit(EXIT_FAILURE, "invalid argument\n");
> +
> +	if (interactive == 1) {
> +		cl = cmdline_stdin_new(main_ctx, "vdpa> ");
> +		if (cl == NULL)
> +			rte_panic("Cannot create cmdline instance\n");
> +		cmdline_interact(cl);
> +		cmdline_stdin_exit(cl);
> +	} else {
> +		for (i = 0; i <  RTE_MIN(MAX_VDPA_SAMPLE_PORTS,
> dev_total);

double " "

BRs,
Xiao